diff --git a/conf/keycloak/docker-compose-dev.yml b/conf/keycloak/docker-compose-dev.yml index 7e57cd7d83c..bdca3b85aa1 100644 --- a/conf/keycloak/docker-compose-dev.yml +++ b/conf/keycloak/docker-compose-dev.yml @@ -31,6 +31,7 @@ services: DATAVERSE_FEATURE_API_BEARER_AUTH_USE_BUILTIN_USER_ON_ID_MATCH: "1" DATAVERSE_MAIL_SYSTEM_EMAIL: "dataverse@localhost" DATAVERSE_MAIL_MTA_HOST: "smtp" + DATAVERSE_API_SIGNING_SECRET: "dev-only-signing-secret-change-me" DATAVERSE_AUTH_OIDC_ENABLED: "1" DATAVERSE_AUTH_OIDC_CLIENT_ID: test DATAVERSE_AUTH_OIDC_CLIENT_SECRET: 94XHrfNRwXsjqTqApRrwWmhDLDHpIYV8 diff --git a/doc/release-notes/fix-url-signing-special-characters.md b/doc/release-notes/fix-url-signing-special-characters.md new file mode 100644 index 00000000000..ba6b4964ff6 --- /dev/null +++ b/doc/release-notes/fix-url-signing-special-characters.md @@ -0,0 +1,74 @@ +### Signed URLs work again for URLs with special characters + +Requesting a signed URL (e.g. via `/api/admin/requestSignedUrl`, used by external tools, the Globus +integration and third-party integrations such as the `rdm-integration` connector) was broken in 6.10 +for URLs whose query contained special characters — most notably persistent IDs such as +`doi:10.5072/FK2/ABC` (which contain `:` and `/`), as well as spaces, percent-encoded values and +non-ASCII characters. In 6.10 the signing step began re-encoding/normalizing the URL (for example +percent-encoding `:` and `/`) before computing the signature, while the request is validated against +the URL the caller actually presents back. The re-encoded signature no longer matched, so validation +failed with 401 "Bad signed URL" authentication errors. + +Signing no longer alters the URL at all: it is signed exactly as provided, character for character. +As part of this, a URL to be signed must not already contain any of the reserved query parameters +`until`, `user`, `method`, `token` (added by the signing itself), `key` or `signed`: + +- `/api/admin/requestSignedUrl` now returns a 400 (Bad Request) naming the offending parameter when + the supplied `url` contains one of them. In 6.10 such parameters were silently stripped, so the + caller received a signature for a different URL than the one submitted; before 6.10 the parameter + was signed into the URL, with undefined results at validation time. +- The `?signed=true` guestbook-response download flow is unaffected: the reserved parameters that + legitimately appear in such a request (`signed` itself, `key` when query-parameter API token + authentication is used, and the four signing parameters when the request was authenticated with an + existing signed URL) are removed from the request URL before the new signed URL is created, as in + 6.10. + +**This restores the URL-signing behavior used before 6.10, so it is compatible with older versions +and with existing integrations.** Clients and connectors that build or consume signed URLs the way +they did before 6.10 keep working unchanged, signatures are computed the same way as before the +regression, and URLs containing special characters validate again. No client-side changes are +required for URLs that follow the documented contract; the one exception is described below - a +`url` that already contains a reserved query parameter is now rejected with a 400 instead of being +silently altered. + +Validation has also been made encoding-agnostic. The signature is now checked against the URL +exactly as presented on the wire first, so the simple contract just works: submit the `url` to +`/api/admin/requestSignedUrl` in exactly the form you will use it - percent-encoded or not - and use +the returned signed URL verbatim. URLs signed in their URL-decoded form and later presented as a +percent-encoded variant also continue to validate (the pre-6.10 behavior, kept for compatibility +with existing clients and with proxies or HTTP libraries that re-encode characters in flight). +Clients whose requests may be re-encoded in flight should therefore prefer submitting the URL in +its URL-decoded form, which is robust in both cases. + +### A signing secret is now required for signed URLs + +Separately from the fix above, Dataverse no longer falls back to a weak signing key when +`dataverse.api.signing-secret` is unset. Previously, with no secret configured, signed URLs were +signed using only the user's API token (or, for a guest, a value derived from the public URL), which +is too weak to be a signing key. A non-empty `dataverse.api.signing-secret` is now required wherever +URLs are signed with a key based on a user's API token: + +- The endpoints that issue a signed URL on request - `/api/admin/requestSignedUrl` and the `POST` + guestbook-response download endpoints under `/api/access` (`datafile/{id}`, `datafiles/{ids}`, + `dataset/{id}` and `dataset/{id}/versions/{versionId}`) - return an error instead of issuing a + weakly-signed URL. +- External tool launches send their callback unsigned (with a warning logged) rather than weakly + signed. Such an unsigned callback only allows anonymous access to public data; it cannot be used to + access draft datasets or restricted files. +- Globus transfers are effectively disabled without a signing secret: the callbacks the + dataverse-globus app relies on require an authenticated, signed request, so Globus upload (and + download of restricted or unpublished files) fails with an authorization error. Installations that + use Globus must set `dataverse.api.signing-secret`. +- The permissions-history CSV download links on the permission management pages are not offered + (a warning is logged). + +Remote and Globus overlay stores are unaffected: they sign with their own per-store secret key, not +`dataverse.api.signing-secret`. + +**Upgrade note:** installations that rely on signed URLs - including the `rdm-integration` connector, +signed guestbook-response downloads, and external tools or Globus transfers that use signed callbacks - +must set `dataverse.api.signing-secret`. See the +[Configuration Guide](https://guides.dataverse.org/en/latest/installation/config.html#dataverse-api-signing-secret). +Treat the value like a password. Because the signing secret is part of the signing key, setting (or +later changing) it invalidates previously issued signed URLs: any existing signed URLs that have not +yet expired will stop working, and clients/integrations will need to request new ones. diff --git a/doc/sphinx-guides/source/api/external-tools.rst b/doc/sphinx-guides/source/api/external-tools.rst index c583c9516cc..5242b87a9c4 100644 --- a/doc/sphinx-guides/source/api/external-tools.rst +++ b/doc/sphinx-guides/source/api/external-tools.rst @@ -178,6 +178,10 @@ The signed URL mechanism is more secure than exposing API tokens and therefore r - For tools invoked via a GET call, Dataverse will include a callback query parameter with a Base64 encoded value. The decoded value is a signed URL that can be called to retrieve a JSON response containing all of the queryParameters and allowedApiCalls specified in the manfiest. - For tools invoked via POST, Dataverse will send a JSON body including the requested queryParameters and allowedApiCalls. Dataverse expects the response to the POST to indicate a redirect which Dataverse will use to open the tool. +.. note:: + + **For Dataverse site administrators:** Signing these URLs requires a non-empty :ref:`dataverse.api.signing-secret` to be configured on the Dataverse installation. Without it, the callback and ``allowedApiCalls`` URLs are sent unsigned (a warning is logged), so tools cannot use them to access draft datasets or restricted files. + .. note:: **For Dataverse site administrators:** When Dataverse is behind a proxy, signed URLs may not work correctly due to protocol mismatches (HTTP vs HTTPS). Please refer to the :ref:`signed-urls-forwarded-proto-header` section to ensure signed URLs work properly in proxy environments. diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index 582891e0861..ee679e6c382 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -9139,7 +9139,7 @@ Signed URLs were developed to support External Tools but may be useful in other This API call allows a Dataverse superUser to generate a signed URL for such scenarios. The JSON input parameter required is an object with the following keys: -- ``url`` - the exact URL to sign, including api version number and all query parameters +- ``url`` - the exact URL to sign, including api version number and all query parameters. Provide it in its URL-decoded form (for example ``persistentId=doi:10.5072/FK2/ABC``): the signature is computed over the URL exactly as provided, and a URL signed in decoded form validates both when used as received and when parts of it get percent-encoded in flight. The URL must not already contain any of the reserved query parameters ``until``, ``user``, ``method``, ``token`` (these are appended by the signing), ``key`` or ``signed``; such a URL is rejected with a 400 (Bad Request) response. - ``timeOut`` - how long in minutes the signature should be valid for, default is 10 minutes - ``httpMethod`` - which HTTP method is required, default is GET - ``user`` - the user identifier for the account associated with this signature, the default is the superuser making the call. The API call will succeed/fail based on whether the specified user has the required permissions. @@ -9154,8 +9154,14 @@ A curl example using allowing access to a dataset's metadata curl -H "X-Dataverse-key:$API_KEY" -H 'Content-Type:application/json' -d "$JSON" "$SERVER_URL/api/admin/requestSignedUrl" -Please see :ref:`dataverse.api.signing-secret` for the configuration option to add a shared secret, enabling extra -security. +Use the returned ``signedUrl`` as received, treating it as an opaque string: do not parse and rebuild it or add query +parameters, as reordering or inserting anything invalidates the signature. If you need a percent-encoded form (for +example a search term containing spaces), append the returned signature parameters to the URL exactly as you +originally constructed it. + +Note that a non-empty :ref:`dataverse.api.signing-secret` must be configured on the server for this endpoint to work: +the key used to sign the URL is created from the user's API token plus the signing-secret, and without the secret the +endpoint returns an error instead of issuing a weakly-signed URL. .. _send-feedback-admin: diff --git a/doc/sphinx-guides/source/developers/globus-api.rst b/doc/sphinx-guides/source/developers/globus-api.rst index eb0eb465315..18e0f7f797f 100644 --- a/doc/sphinx-guides/source/developers/globus-api.rst +++ b/doc/sphinx-guides/source/developers/globus-api.rst @@ -78,6 +78,7 @@ Note that while Dataverse will not add files that violate the size or quota rule they intend to transfer before submitting a transfer request to Globus. The getDatasetMetadata and getFileListing URLs are just signed versions of the standard Dataset metadata and file listing API calls. The other two are Globus specific. +Note that these URLs are only signed when a non-empty :ref:`dataverse.api.signing-secret` is configured on the Dataverse installation. Without it the callbacks are sent unsigned (a warning is logged), and because the Globus callback endpoints (``globusUploadParameters``, ``requestGlobusUploadPaths``, ``addGlobusFiles``, and the download equivalents for restricted or unpublished files) require an authenticated, signed request, the Globus transfer flow fails with an authorization error; only anonymous download of public files can proceed. In practice, Globus transfer support requires ``dataverse.api.signing-secret`` to be set. If called for a dataset using a store that is configured with a remote Globus endpoint(s), the return response is similar but the response includes a the "managed" parameter will be false, the "endpoint" parameter is replaced with a JSON array of "referenceEndpointsWithPaths" and the @@ -112,7 +113,7 @@ Once the user identifies which files are to be added, the requestGlobusTransferP curl -H "X-Dataverse-key:$API_TOKEN" -H "Content-type:application/json" -X POST -d "$JSON_DATA" "$SERVER_URL/api/datasets/:persistentId/requestGlobusUploadPaths?persistentId=$PERSISTENT_IDENTIFIER" -Note that when using the dataverse-globus app or the return from the previous call, the URL for this call will be signed and no API_TOKEN is needed. +Note that when using the dataverse-globus app or the return from the previous call, the URL for this call will be signed (provided a non-empty :ref:`dataverse.api.signing-secret` is configured, as noted above) and no API_TOKEN is needed. In the managed case, the JSON body sent must include the id of the Globus user that will perform the transfer and the number of files that will be transferred: diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index 841f28391c2..4dbf0681016 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -3364,15 +3364,18 @@ are time limited and only allow the action of the API call in the URL. See :ref: :ref:`api-native-signed-url` for more details. The key used to sign a URL is created from the API token of the creating user plus a signing-secret provided by an administrator. -**Using a signing-secret is highly recommended.** This setting defaults to an empty string. Using a non-empty -signing-secret makes it impossible for someone who knows an API token from forging signed URLs and provides extra security by -making the overall signing key longer. +**A non-empty signing-secret is required to request signed URLs through the API.** If it is not configured, the +``/api/admin/requestSignedUrl`` endpoint (see :ref:`api-native-signed-url`) and the ``POST`` guestbook-response download +endpoints under ``/api/access`` return an error instead of issuing a weakly-signed URL, and internally generated links +(external tool callbacks, Globus transfers, the permissions-history CSV downloads) are sent unsigned or omitted, with a +warning logged. (The setting otherwise defaults to an empty string.) A non-empty signing-secret makes it impossible for +someone who only knows an API token to forge signed URLs, and provides extra security by making the overall signing key longer. **WARNING**: *Since the signing-secret is sensitive, you should treat it like a password.* *See* :ref:`secure-password-storage` *to learn about ways to safeguard it.* -Can also be set via any `supported MicroProfile Config API source`_, e.g. the environment variable ``DATAVERSE_API_SIGNATURE_SECRET`` (although you shouldn't use environment variables for passwords) . +Can also be set via any `supported MicroProfile Config API source`_, e.g. the environment variable ``DATAVERSE_API_SIGNING_SECRET`` (although you shouldn't use environment variables for passwords) . .. _dataverse.api.allow-incomplete-metadata: diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index c176597c990..4b4d9168a04 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -60,6 +60,7 @@ services: -Ddataverse.pid.fake.label=FakeDOIProvider -Ddataverse.pid.fake.authority=10.5072 -Ddataverse.pid.fake.shoulder=FK2/ + -Ddataverse.api.signing-secret=dev-only-signing-secret-change-me -Ddataverse.cors.origin=* \ -Ddataverse.cors.methods=GET,POST,PUT,DELETE,OPTIONS \ -Ddataverse.cors.headers.allow=range,content-type,x-dataverse-key,accept \ diff --git a/docker/compose/demo/compose.yml b/docker/compose/demo/compose.yml index 8b41e408616..070ef7d9f04 100644 --- a/docker/compose/demo/compose.yml +++ b/docker/compose/demo/compose.yml @@ -16,6 +16,10 @@ services: DATAVERSE_FEATURE_API_BEARER_AUTH: "1" DATAVERSE_MAIL_SYSTEM_EMAIL: "Demo Dataverse " DATAVERSE_MAIL_MTA_HOST: "smtp" + # Required for signed URLs (external tools, Globus transfers, signed guestbook downloads, + # rdm-integration). Change this to a private value before exposing the instance - with this + # well-known default, anyone can forge signed URLs. See the Configuration Guide. + DATAVERSE_API_SIGNING_SECRET: "demo-only-signing-secret-change-me" JVM_ARGS: -Ddataverse.files.storage-driver-id=file1 -Ddataverse.files.file1.type=file -Ddataverse.files.file1.label=Filesystem diff --git a/src/main/java/edu/harvard/iq/dataverse/ManageFilePermissionsPage.java b/src/main/java/edu/harvard/iq/dataverse/ManageFilePermissionsPage.java index ee687305584..4e67919959f 100644 --- a/src/main/java/edu/harvard/iq/dataverse/ManageFilePermissionsPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/ManageFilePermissionsPage.java @@ -22,7 +22,6 @@ import edu.harvard.iq.dataverse.engine.command.exception.PermissionException; import edu.harvard.iq.dataverse.engine.command.impl.AssignRoleCommand; import edu.harvard.iq.dataverse.engine.command.impl.RevokeRoleCommand; -import edu.harvard.iq.dataverse.settings.JvmSettings; import edu.harvard.iq.dataverse.util.BundleUtil; import edu.harvard.iq.dataverse.util.DateUtil; import edu.harvard.iq.dataverse.util.JsfHelper; @@ -640,9 +639,11 @@ public String getSignedUrlForRAHistoryCsv() { key = apiToken.getTokenString(); } } - key = JvmSettings.API_SIGNING_SECRET.lookupOptional().orElse("") + key; - if(key.length() >= 36) { - return UrlSignerUtil.signUrl(fullApiPath, 10, userId, "GET", key); + if (key != null) { + if (UrlSignerUtil.isSigningSecretConfigured()) { + return UrlSignerUtil.signUrlWithApiKey(fullApiPath, 10, userId, "GET", key); + } + logger.warning("Cannot sign the permissions-history CSV link: no signing secret configured (dataverse.api.signing-secret). The download link will not be shown."); } } catch (Exception e) { logger.log(Level.SEVERE, "Error generating signed URL for permissions history CSV: " + e.getMessage(), e); diff --git a/src/main/java/edu/harvard/iq/dataverse/ManagePermissionsPage.java b/src/main/java/edu/harvard/iq/dataverse/ManagePermissionsPage.java index f5cd859e7ac..778e28d90cf 100644 --- a/src/main/java/edu/harvard/iq/dataverse/ManagePermissionsPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/ManagePermissionsPage.java @@ -20,7 +20,6 @@ import edu.harvard.iq.dataverse.engine.command.impl.CreateRoleCommand; import edu.harvard.iq.dataverse.engine.command.impl.RevokeRoleCommand; import edu.harvard.iq.dataverse.engine.command.impl.UpdateDataverseDefaultContributorRoleCommand; -import edu.harvard.iq.dataverse.settings.JvmSettings; import edu.harvard.iq.dataverse.util.BundleUtil; import edu.harvard.iq.dataverse.util.JsfHelper; import static edu.harvard.iq.dataverse.util.JsfHelper.JH; @@ -736,9 +735,11 @@ public String getSignedUrlForRAHistoryCsv() { key = apiToken.getTokenString(); } } - key = JvmSettings.API_SIGNING_SECRET.lookupOptional().orElse("") + key; - if(key.length() >= 36) { - return UrlSignerUtil.signUrl(fullApiPath, 10, userId, "GET", key); + if (key != null) { + if (UrlSignerUtil.isSigningSecretConfigured()) { + return UrlSignerUtil.signUrlWithApiKey(fullApiPath, 10, userId, "GET", key); + } + logger.warning("Cannot sign the permissions-history CSV link: no signing secret configured (dataverse.api.signing-secret). The download link will not be shown."); } } catch (Exception e) { logger.log(Level.SEVERE, "Error generating signed URL for permissions history CSV: " + e.getMessage(), e); diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Access.java b/src/main/java/edu/harvard/iq/dataverse/api/Access.java index 9b6beb999ae..ea46c14d12d 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Access.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Access.java @@ -29,7 +29,6 @@ import edu.harvard.iq.dataverse.makedatacount.MakeDataCountLoggingServiceBean; import edu.harvard.iq.dataverse.makedatacount.MakeDataCountLoggingServiceBean.MakeDataCountEntry; import edu.harvard.iq.dataverse.mydata.Pager; -import edu.harvard.iq.dataverse.settings.JvmSettings; import edu.harvard.iq.dataverse.settings.SettingsServiceBean; import edu.harvard.iq.dataverse.util.*; import edu.harvard.iq.dataverse.util.json.JsonParseException; @@ -462,6 +461,10 @@ public Response datafileWithGuestbookResponse(@Context ContainerRequestContext c @RequestBody(description = "Guestbook response JSON for the requested data file.") String jsonBody) { + Response requireSecretError = requireSigningSecretForSignedUrl(); + if (requireSecretError != null) { + return requireSecretError; + } DataverseRequest req = createDataverseRequest(getRequestUser(crc)); fileId = normalizeFileId(fileId, req); return processDatafileWithGuestbookResponse(crc, req, headers, fileId, uriInfo, gbrecs, jsonBody); @@ -496,6 +499,22 @@ private List getGuestbookIdFromDatafile(DataFile df) { return df != null && df.getOwner() != null && df.getOwner().getGuestbook() != null ? List.of(df.getOwner().getGuestbook().getId().toString()) : List.of(); } + // Gate for the endpoints whose outcome is a signed download URL. It must run before + // processDatafileWithGuestbookResponse so no guestbook-response or MakeDataCount records are + // persisted for a download URL that is never issued - but it must NOT sit inside that method, + // which is also called purely for its guestbook side effects (with the response discarded) by + // endpoints that never issue signed URLs. Without the secret the key would be only the user's + // API token (or, for a guest, a guessable value derived from the URL), which is too weak. The + // response is reachable by unauthenticated users, so the message stays generic and the + // actionable detail goes to the server log. Mirrors Admin.getSignedUrl. + private Response requireSigningSecretForSignedUrl() { + if (UrlSignerUtil.isSigningSecretConfigured()) { + return null; + } + logger.warning("Cannot issue a signed download URL: no signing secret configured. Please set the dataverse.api.signing-secret JVM option."); + return error(INTERNAL_SERVER_ERROR, "Signed URLs are not available on this server."); + } + // Process the guestbook response from JSON and return a signedUrl to the matching GET call private Response processDatafileWithGuestbookResponse(ContainerRequestContext crc, DataverseRequest req, HttpHeaders headers, String fileIds, UriInfo uriInfo, boolean gbrecs, String jsonBody) { @@ -582,6 +601,21 @@ private Map getDatafilesMap(DataverseRequest req, String fileIds } private Response returnSignedUrl(ContainerRequestContext crc, UriInfo uriInfo, User user, String id, String gbrids) { + // Record the guestbook-response id for the side-effect-only callers + // (datafileBundleWithGuestbookResponse, postDownloadDatafiles) that discard the Response we + // return and then read this property to drive the actual download. This must happen even when + // no signing secret is configured and we bail out below, otherwise those downloads lose the + // guestbook response that processDatafileWithGuestbookResponse just saved (leading to a + // spurious "guestbookResponseMissing" 400 or a duplicate guestbook/MakeDataCount entry). + crc.setProperty("gbrids", gbrids); + // The signed-URL endpoints already check this before any guestbook side effects (see + // requireSigningSecretForSignedUrl). This check protects the callers that invoke + // processDatafileWithGuestbookResponse only for its guestbook side effects and discard this + // response: on a server without a signing secret they must not trip the IllegalStateException + // in signUrlWithApiKey below. No warning is logged here - for those callers this is routine. + if (!UrlSignerUtil.isSigningSecretConfigured()) { + return error(INTERNAL_SERVER_ERROR, "Signed URLs are not available on this server."); + } // Create the signed URL String userIdentifier = null; String key = null; @@ -613,12 +647,16 @@ private Response returnSignedUrl(ContainerRequestContext crc, UriInfo uriInfo, U builder.replaceQueryParam("gbrids", gbrids); } builder.replaceQueryParam("persistentId", null); // remove this as a parm and add the id to the path - crc.setProperty("gbrids", gbrids); String baseUrlEncoded = builder.build().toString(); String baseUrl = URLDecoder.decode(baseUrlEncoded, StandardCharsets.UTF_8); baseUrl = baseUrl.replace(":persistentId", id); - key = JvmSettings.API_SIGNING_SECRET.lookupOptional().orElse("") + key; - String signedUrl = UrlSignerUtil.signUrl(baseUrl, GUESTBOOK_RESPONSE_SIGNEDURL_TIMEOUT_MINUTES, userIdentifier, "GET", key); + // The request URI this URL is built from carries params that must not end up in the signed + // URL: "signed" (which selected this flow - re-signing it would loop), "key" (the query-param + // API token - a credential), and, when the request was itself authenticated with a signed URL, + // the four signing params. signUrlWithApiKey refuses a URL containing signing params rather + // than fixing it, so this caller strips all of them first. + baseUrl = UrlSignerUtil.stripReservedParameters(baseUrl); + String signedUrl = UrlSignerUtil.signUrlWithApiKey(baseUrl, GUESTBOOK_RESPONSE_SIGNEDURL_TIMEOUT_MINUTES, userIdentifier, "GET", key); return ok(JsonUtil.createObjectBuilder().add(URLTokenUtil.SIGNED_URL, signedUrl)); } @@ -986,6 +1024,10 @@ public Response downloadAllFromLatestWithGuestbookResponse(@Context ContainerReq @Context UriInfo uriInfo, @Context HttpHeaders headers, @Context HttpServletResponse response, @RequestBody(description = "Guestbook response JSON for the dataset file download.") String jsonBody) throws WebApplicationException { + Response requireSecretError = requireSigningSecretForSignedUrl(); + if (requireSecretError != null) { + return requireSecretError; + } try { User user = getRequestUser(crc); DataverseRequest req = createDataverseRequest(user); @@ -1080,6 +1122,10 @@ public Response downloadAllFromVersionWithGuestbookResponse(@Context ContainerRe @RequestBody(description = "Guestbook response JSON for the dataset version file download.") String jsonBody, @Context UriInfo uriInfo, @Context HttpHeaders headers, @Context HttpServletResponse response) throws WebApplicationException { + Response requireSecretError = requireSigningSecretForSignedUrl(); + if (requireSecretError != null) { + return requireSecretError; + } try { DataverseRequest req = createDataverseRequest(getRequestUser(crc)); DatasetVersion dsv = getDatasetVersionFromVersion(crc, datasetIdOrPersistentId, versionId); @@ -1187,6 +1233,10 @@ public Response datafilesWithGuestbookResponse(@Context ContainerRequestContext @RequestBody(description = "Guestbook response JSON for the selected data file download.") String jsonBody) throws WebApplicationException { + Response requireSecretError = requireSigningSecretForSignedUrl(); + if (requireSecretError != null) { + return requireSecretError; + } DataverseRequest req = createDataverseRequest(getRequestUser(crc)); return processDatafileWithGuestbookResponse(crc, req, headers, fileIds, uriInfo, gbrecs, jsonBody); } diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Admin.java b/src/main/java/edu/harvard/iq/dataverse/api/Admin.java index 38209ebdd94..d353b7683d9 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Admin.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Admin.java @@ -17,7 +17,6 @@ import edu.harvard.iq.dataverse.DvObjectServiceBean; import edu.harvard.iq.dataverse.FileMetadata; import edu.harvard.iq.dataverse.api.auth.AuthRequired; -import edu.harvard.iq.dataverse.settings.JvmSettings; import edu.harvard.iq.dataverse.settings.SettingsValidationException; import edu.harvard.iq.dataverse.util.StringUtil; import edu.harvard.iq.dataverse.util.cache.CacheFactoryBean; @@ -2726,36 +2725,58 @@ public Response getSignedUrl(@Context ContainerRequestContext crc, if (superuser == null || !superuser.isSuperuser()) { return error(Response.Status.FORBIDDEN, "Requesting signed URLs is restricted to superusers."); } - - String userId = urlInfo.getString("user"); - String key=null; - if (userId != null) { - AuthenticatedUser user = authSvc.getAuthenticatedUser(userId); - // If a user param was sent, we sign the URL for them, otherwise on behalf of - // the superuser who made this api call - if (user != null) { - ApiToken apiToken = authSvc.findApiTokenByUser(user); - if (apiToken != null && !apiToken.isExpired() && !apiToken.isDisabled()) { - key = apiToken.getTokenString(); - } - } else { - userId = superuser.getUserIdentifier(); - // We ~know this exists - the superuser just used it and it was unexpired/not - // disabled. (ToDo - if we want this to work with workflow tokens (or as a - // signed URL), we should do more checking as for the user above)) - key = authSvc.findApiTokenByUser(superuser).getTokenString(); + + // Require a signing secret: without it the key is only the user's API token, which is too weak. + if (!UrlSignerUtil.isSigningSecretConfigured()) { + return error(Response.Status.INTERNAL_SERVER_ERROR, + "Requesting signed URLs requires a signing secret to be configured. Please set the dataverse.api.signing-secret JVM option."); + } + + // "url" is required; "user" defaults to the superuser making the call (see the docs for this + // endpoint). Use the defaulted accessors: JsonObject.getString(name) throws NullPointerException + // when the key is absent. + String baseUrl = urlInfo.getString("url", null); + if (baseUrl == null) { + return error(Response.Status.BAD_REQUEST, "Required parameter 'url' is missing."); + } + // Reject rather than silently rewrite (in 6.10 reserved params were quietly stripped, so the + // caller got back a signature for a different URL than requested). The four signing params + // are added by the signing itself; "key" and "signed" are reserved at the request level: a + // signed "key" would bake a credential param into the URL and "signed=true" would make the + // signed URL return yet another signed URL instead of the resource. + String reserved = UrlSignerUtil.findReservedParameter(baseUrl, UrlSignerUtil.reservedParameters); + if (reserved != null) { + return error(Response.Status.BAD_REQUEST, + "The url to sign must not contain the reserved query parameter '" + reserved + "'."); + } + String userId = urlInfo.getString("user", null); + + String key = null; + AuthenticatedUser signingUser = (userId != null) ? authSvc.getAuthenticatedUser(userId) : null; + if (signingUser != null) { + // A known user was requested: sign the URL for them. + ApiToken apiToken = authSvc.findApiTokenByUser(signingUser); + if (apiToken != null && !apiToken.isExpired() && !apiToken.isDisabled()) { + key = apiToken.getTokenString(); } - if (key == null) { - return error(Response.Status.CONFLICT, "Do not have a valid user with apiToken"); + } else { + // No user param, or an unknown one: sign on behalf of the superuser who made this API call. + userId = superuser.getUserIdentifier(); + // The superuser just authenticated, but that does not guarantee an API token exists (e.g. + // bearer-token or session auth), so null-check rather than dereference blindly. + ApiToken apiToken = authSvc.findApiTokenByUser(superuser); + if (apiToken != null) { + key = apiToken.getTokenString(); } - key = JvmSettings.API_SIGNING_SECRET.lookupOptional().orElse("") + key; } - - String baseUrl = urlInfo.getString("url"); + if (key == null) { + return error(Response.Status.CONFLICT, "Do not have a valid user with apiToken"); + } + int timeout = urlInfo.getInt(URLTokenUtil.TIMEOUT, 10); String method = urlInfo.getString(URLTokenUtil.HTTP_METHOD, "GET"); - - String signedUrl = UrlSignerUtil.signUrl(baseUrl, timeout, userId, method, key); + + String signedUrl = UrlSignerUtil.signUrlWithApiKey(baseUrl, timeout, userId, method, key); return ok(JsonUtil.createObjectBuilder().add(URLTokenUtil.SIGNED_URL, signedUrl)); } diff --git a/src/main/java/edu/harvard/iq/dataverse/api/auth/SignedUrlAuthMechanism.java b/src/main/java/edu/harvard/iq/dataverse/api/auth/SignedUrlAuthMechanism.java index d21d91a07c2..da57ac97b8b 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/auth/SignedUrlAuthMechanism.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/auth/SignedUrlAuthMechanism.java @@ -4,7 +4,6 @@ import edu.harvard.iq.dataverse.authorization.users.*; import edu.harvard.iq.dataverse.privateurl.PrivateUrl; import edu.harvard.iq.dataverse.privateurl.PrivateUrlServiceBean; -import edu.harvard.iq.dataverse.settings.JvmSettings; import edu.harvard.iq.dataverse.util.UrlSignerUtil; import jakarta.inject.Inject; @@ -60,6 +59,21 @@ private User getAuthenticatedUserFromSignedUrl(ContainerRequestContext container // If User is Guest we can return a generic guest user with key made from URI UriInfo uriInfo = containerRequestContext.getUriInfo(); String userId = uriInfo.getQueryParameters().getFirst(SIGNED_URL_USER); + if (userId == null) { + // A token param was present (that is why this mechanism ran) but no user param: this can + // never be a URL we signed, and dereferencing userId below would throw a NullPointerException. + // This check runs before the signing-secret warning below so that arbitrary requests that + // merely carry a token param cannot flood the log on installations without a secret. + return null; + } + // Without a signing secret we never issue signed URLs (signUrlWithApiKey refuses on the sign + // side), so we must not accept them here either. Otherwise a bare API token - or, for a guest, + // the public request URL - would be enough to forge a URL whose signature validates against the + // "" + token key computed below. Reject so findUserFromRequest returns the standard 401. + if (!UrlSignerUtil.isSigningSecretConfigured()) { + logger.warning("Rejecting signed URL authentication: no signing secret configured (dataverse.api.signing-secret)."); + return null; + } User targetUser = null; ApiToken userApiToken = null; if (userId.equalsIgnoreCase("guest")) { @@ -70,32 +84,65 @@ private User getAuthenticatedUserFromSignedUrl(ContainerRequestContext container targetUser = authSvc.getAuthenticatedUser(userId); userApiToken = authSvc.findApiTokenByUser((AuthenticatedUser) targetUser); } else { - PrivateUrl privateUrl = privateUrlSvc.getPrivateUrlFromDatasetId(Long.parseLong(userId.substring(PrivateUrlUser.PREFIX.length()))); + // The user param is attacker-controlled: a non-numeric suffix or a dataset without a + // private URL must yield the standard 401, not an unhandled 500. + PrivateUrl privateUrl = null; + try { + privateUrl = privateUrlSvc.getPrivateUrlFromDatasetId(Long.parseLong(userId.substring(PrivateUrlUser.PREFIX.length()))); + } catch (NumberFormatException e) { + return null; + } + if (privateUrl == null) { + return null; + } userApiToken = new ApiToken(); userApiToken.setTokenString(privateUrl.getToken()); targetUser = privateUrlSvc.getPrivateUrlUserFromToken(privateUrl.getToken()); } if (targetUser != null && userApiToken != null) { - String signedUrl = URLDecoder.decode(uriInfo.getRequestUri().toString(), StandardCharsets.UTF_8); - - logger.fine("Original URL: " + containerRequestContext.getUriInfo().getRequestUri().toString()); + String rawUrl = uriInfo.getRequestUri().toString(); + logger.fine("Original URL: " + rawUrl); String forwardedProto = containerRequestContext.getHeaderString("X-Forwarded-Proto"); logger.fine("X-Forwarded-Proto is: " + forwardedProto); - - - if (forwardedProto != null && !forwardedProto.isEmpty()) { - if ("https".equalsIgnoreCase(forwardedProto) && signedUrl.toLowerCase().startsWith("http:")) { - signedUrl = "https" + signedUrl.substring(4); - } - } + rawUrl = applyForwardedProto(rawUrl, forwardedProto); String requestMethod = containerRequestContext.getMethod(); - String signedUrlSigningKey = JvmSettings.API_SIGNING_SECRET.lookupOptional().orElse("") + userApiToken.getTokenString(); - boolean isSignedUrlValid = UrlSignerUtil.isValidUrl(signedUrl, userId, requestMethod, signedUrlSigningKey); - if (isSignedUrlValid) { + String signedUrlSigningKey = UrlSignerUtil.getApiSigningKey(userApiToken.getTokenString()); + if (isSignedUrlValid(rawUrl, userId, requestMethod, signedUrlSigningKey)) { user = targetUser; } } return user; } + + // Primary contract: the signature is checked against the exact bytes on the wire, so a URL that + // was signed in the very form the client presents it - percent-escapes included - works verbatim, + // with no client-side decoding or reconstruction. The fallback (the only behavior before 6.11) + // covers URLs signed in their URL-decoded form and presented as an encoded variant; it also + // absorbs clients and proxies that re-encode characters in flight (every signed URL carries ':' + // in its "until" timestamp, a favorite of normalizing HTTP stacks). + private static boolean isSignedUrlValid(String rawUrl, String userId, String requestMethod, String signingKey) { + if (UrlSignerUtil.isValidUrl(rawUrl, userId, requestMethod, signingKey)) { + return true; + } + try { + String decoded = URLDecoder.decode(rawUrl, StandardCharsets.UTF_8); + // When decoding is the identity (no %-escapes or '+'), the fallback would hash the same + // bytes again; skip it. + return !decoded.equals(rawUrl) && UrlSignerUtil.isValidUrl(decoded, userId, requestMethod, signingKey); + } catch (IllegalArgumentException e) { + // Not URL-decodable (e.g. a bare '%'): there is no decoded variant to check against. + logger.fine("Signed URL is not URL-decodable, skipping the decoded-form check: " + e.getMessage()); + return false; + } + } + + // Behind a TLS-terminating proxy the request URI is http:// while the URL was signed as + // https://; restore the original protocol before validating the signature. + private static String applyForwardedProto(String signedUrl, String forwardedProto) { + if ("https".equalsIgnoreCase(forwardedProto) && signedUrl.regionMatches(true, 0, "http:", 0, 5)) { + return "https" + signedUrl.substring(4); + } + return signedUrl; + } } diff --git a/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandler.java b/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandler.java index e7ae451cacf..ed770f1934b 100644 --- a/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandler.java +++ b/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandler.java @@ -4,7 +4,6 @@ import edu.harvard.iq.dataverse.Dataset; import edu.harvard.iq.dataverse.FileMetadata; import edu.harvard.iq.dataverse.authorization.users.ApiToken; -import edu.harvard.iq.dataverse.settings.JvmSettings; import edu.harvard.iq.dataverse.util.SystemConfig; import edu.harvard.iq.dataverse.util.URLTokenUtil; @@ -111,8 +110,8 @@ public String handleRequest(boolean preview) { + externalTool.getId(); } if (apiToken != null) { - callback = UrlSignerUtil.signUrl(callback, 5, apiToken.getAuthenticatedUser().getUserIdentifier(), HttpMethod.GET, - JvmSettings.API_SIGNING_SECRET.lookupOptional().orElse("") + apiToken.getTokenString()); + callback = UrlSignerUtil.trySignUrlWithApiKey(callback, 5, apiToken.getAuthenticatedUser().getUserIdentifier(), + HttpMethod.GET, apiToken.getTokenString(), "external tool callback"); } paramsString= "?callback=" + Base64.getEncoder().encodeToString(StringUtils.getBytesUtf8(callback)); if (getLocaleCode() != null) { diff --git a/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java index f204dd1f156..fbc833ebb9f 100644 --- a/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java @@ -776,13 +776,15 @@ public String getGlobusAppUrlForDataset(Dataset d, boolean upload, List reservedParameters = List.of(SIGNED_URL_UNTIL, SIGNED_URL_USER, SIGNED_URL_METHOD, SIGNED_URL_TOKEN, SIGNED_URL_KEY, SIGNED_URL_SIGNED); + public static final String SIGNED_URL_KEY="key"; // reserved at the Dataverse request level: the legacy query-param API token - a credential that must never be signed into a URL + public static final String SIGNED_URL_SIGNED="signed"; // reserved at the Dataverse request level: requests a (re)signed URL as the response - signing it into a URL would cause a loop of signing + /** + * The four query parameters the signing algorithm itself appends. {@link #signUrl} throws if the + * base URL already contains one of them, rather than signing a different URL than the caller + * intended. + */ + public static final List signingParameters = List.of(SIGNED_URL_UNTIL, SIGNED_URL_USER, SIGNED_URL_METHOD, SIGNED_URL_TOKEN); + /** + * The signing parameters plus the Dataverse request-level parameters {@code key} and + * {@code signed}, none of which may appear in a URL being signed. Callers that build the URL to + * sign out of an incoming request URI - which legitimately carries such parameters - must remove + * them first, e.g. with {@link #stripReservedParameters}; {@code signUrl} itself never rewrites + * the URL it is given. + */ + public static final List reservedParameters = Stream.concat( + signingParameters.stream(), Stream.of(SIGNED_URL_KEY, SIGNED_URL_SIGNED)).toList(); /** * - * @param baseUrl - the URL to sign - cannot contain query params - * "until","user", "method", or "token" + * @param baseUrl - the URL to sign - must not contain the query params + * "until","user", "method", or "token" (this method throws if it does) * @param timeout - how many minutes to make the URL valid for (note - time skew * between the creator and receiver could affect the validation * @param user - a string representing the user - should be understood by the @@ -42,21 +56,21 @@ public class UrlSignerUtil { * this could be an APIKey (when sending URL to a tool that will * use it to retrieve info from Dataverse) * @return - the signed URL + * @throws IllegalArgumentException if the base URL already contains one of the four query + * parameters this method appends ({@link #signingParameters}) */ public static String signUrl(String baseUrl, Integer timeout, String user, String method, String key) { - // check for reserved parameter names ("until","user", "method", or "token") - String[] urlQP = baseUrl.split("\\?"); - if (urlQP.length > 1) { - try { - URIBuilder uriBuilder = new URIBuilder(baseUrl); - List params = uriBuilder.getQueryParams(); - params.removeIf(pair -> reservedParameters.contains(pair.getName())); - uriBuilder.setParameters(params); - baseUrl = uriBuilder.build().toString(); - } catch (URISyntaxException e) { - logger.severe("Invalid URL for signing: " + baseUrl + " " + e.getMessage()); - } + // The URL is signed exactly as provided and never rewritten here (the pre-6.10 behavior): + // validation reconstructs the signing string from the URL-decoded request, so re-encoding or + // dropping params would change the signed bytes and the signature would no longer match. A + // base URL that already contains one of the four params this method appends is therefore a + // caller bug - the stray param would sit inside the signed bytes and be enforced at + // validation time as if it had been added here. Fail loudly instead of fixing it silently. + String clash = findReservedParameter(baseUrl, signingParameters); + if (clash != null) { + throw new IllegalArgumentException( + "The URL to sign must not already contain the reserved parameter '" + clash + "': " + baseUrl); } boolean firstParam = !baseUrl.contains("?"); StringBuilder signedUrlBuilder = new StringBuilder(baseUrl); @@ -87,6 +101,150 @@ public static String signUrl(String baseUrl, Integer timeout, String user, Strin return signedUrl; } + /** + * The configured API signing secret ({@code dataverse.api.signing-secret}), or an empty string if + * unset. This is the single definition of what "the signing secret" is; everything else derives + * from it so the sign side and the validation side cannot drift apart. + */ + private static String signingSecret() { + return JvmSettings.API_SIGNING_SECRET.lookupOptional().orElse(""); + } + + /** + * Whether a non-empty API signing secret ({@code dataverse.api.signing-secret}) is configured. + * Every signed URL whose key is derived from a user's API token must be guarded by this: without + * the secret the signing key would be only the caller-supplied value (for a guest, even a value + * derived from the public URL), which is too weak. Callers either refuse the request or skip + * signing when this returns false, so a weakly-signed URL is never emitted. + */ + public static boolean isSigningSecretConfigured() { + return !signingSecret().isEmpty(); + } + + /** + * Composes the full signing key (the configured API signing secret + the given per-user key, + * typically a user's API token). This is the one place that defines how the two are combined, so + * the signing side ({@link #signUrlWithApiKey}) and the validation side (SignedUrlAuthMechanism) + * stay in lockstep. + */ + public static String getApiSigningKey(String apiKey) { + return signingSecret() + apiKey; + } + + /** + * Signs a URL using the configured API signing secret prepended to the given per-user key + * (typically the user's API token). This is the single place that combines the server-side + * signing secret with a user key, so every API-token-based signed URL is produced the same way. + * + *

Stores that sign with their own per-store secret (the remote and Globus overlay stores) are + * the exception and must keep calling {@link #signUrl} directly with that secret. + * + * @throws IllegalStateException if no signing secret is configured - callers should normally + * guard with {@link #isSigningSecretConfigured()} first, or use + * {@link #trySignUrlWithApiKey} to degrade to an unsigned URL + */ + public static String signUrlWithApiKey(String baseUrl, Integer timeout, String user, String method, String apiKey) { + if (!isSigningSecretConfigured()) { + throw new IllegalStateException( + "Cannot sign a URL: no signing secret is configured. Please set the dataverse.api.signing-secret JVM option."); + } + return signUrl(baseUrl, timeout, user, method, getApiSigningKey(apiKey)); + } + + /** + * Signs the URL like {@link #signUrlWithApiKey} when a signing secret is configured; otherwise logs + * a warning naming {@code context} and returns the URL unsigned. This is the single home for the + * "sign if we can, otherwise degrade to an unsigned URL" policy used by the internal callback + * signers (external tools, Globus, allowed API calls), so a caller can never forget the guard and + * trip the {@link IllegalStateException} at runtime. + */ + public static String trySignUrlWithApiKey(String baseUrl, Integer timeout, String user, String method, String apiKey, String context) { + if (!isSigningSecretConfigured()) { + logger.log(Level.WARNING, "Cannot sign {0}: no signing secret configured (dataverse.api.signing-secret). Sending an unsigned URL.", context); + return baseUrl; + } + return signUrlWithApiKey(baseUrl, timeout, user, method, apiKey); + } + + /** + * Caller-side helper that removes the reserved parameters ({@link #reservedParameters}) from a + * URL. It is for callers that build the URL to sign out of an incoming request URI - the + * {@code signed=true} download flow in Access - where the request legitimately carries such + * parameters ({@code signed=true} itself, {@code key} when query-param API token auth was used, + * and even the four signing parameters when the request was authenticated with an existing + * signed URL). {@link #signUrl} never alters the URL it is given, so these must be removed + * before signing. + * + *

The removal preserves the exact bytes of the path, of every other parameter - including + * empty segments such as {@code ?&a=b} or {@code a=b&&} - and of any fragment (unlike + * URIBuilder, which would re-encode and break the MAC). Note that while fragments are preserved + * byte-for-byte, a fragment is never sent to the server, so a fragment-bearing URL cannot + * produce a signed URL that validates - as was the case before 6.10. + */ + public static String stripReservedParameters(String baseUrl) { + // Split off the fragment first: everything from the first '#' on is the fragment (a '?' + // inside it is not a query) and survives the query surgery byte-for-byte, even when the + // fragment is attached to a reserved parameter that gets stripped. + String fragment = ""; + String prefix = baseUrl; + int fragmentStart = baseUrl.indexOf('#'); + if (fragmentStart >= 0) { + fragment = baseUrl.substring(fragmentStart); + prefix = baseUrl.substring(0, fragmentStart); + } + int queryStart = prefix.indexOf('?'); + if (queryStart < 0) { + return baseUrl; + } + String path = prefix.substring(0, queryStart); + String query = prefix.substring(queryStart + 1); + StringBuilder kept = new StringBuilder(); + boolean anyKept = false; + // limit -1 so empty segments (leading '&', '&&', trailing '&') are preserved, not collapsed + for (String pair : query.split("&", -1)) { + int equals = pair.indexOf('='); + String name = (equals < 0) ? pair : pair.substring(0, equals); + if (reservedParameters.contains(name)) { + continue; + } + if (anyKept) { + kept.append('&'); + } + kept.append(pair); + anyKept = true; + } + return anyKept ? path + "?" + kept + fragment : path + fragment; + } + + /** + * Returns the name of the first query parameter of {@code url} whose name is one of + * {@code parameterNames}, or null if none is present. Parameter names are matched the same way + * {@link #stripReservedParameters} matches them (query up to any {@code #} fragment, segments + * split on {@code &}, name being everything before the first {@code =}). Callers that accept a + * URL to sign from outside (e.g. the requestSignedUrl API) can use this with + * {@link #reservedParameters} to reject the URL with a helpful message instead of tripping the + * {@link IllegalArgumentException} in {@link #signUrl}. + */ + public static String findReservedParameter(String url, List parameterNames) { + String prefix = url; + int fragmentStart = url.indexOf('#'); + if (fragmentStart >= 0) { + prefix = url.substring(0, fragmentStart); + } + int queryStart = prefix.indexOf('?'); + if (queryStart < 0) { + return null; + } + for (String pair : prefix.substring(queryStart + 1).split("&", -1)) { + int equals = pair.indexOf('='); + String name = (equals < 0) ? pair : pair.substring(0, equals); + if (parameterNames.contains(name)) { + return name; + } + } + return null; + } + /** * This method will only return true if the URL and parameters except the * "token" are unchanged from the original/match the values sent to this method, diff --git a/src/main/webapp/permissions-manage-files.xhtml b/src/main/webapp/permissions-manage-files.xhtml index 045376c14c4..e37a54ef862 100644 --- a/src/main/webapp/permissions-manage-files.xhtml +++ b/src/main/webapp/permissions-manage-files.xhtml @@ -234,6 +234,7 @@

#{bundle['dataverse.permissions.history.download']} diff --git a/src/main/webapp/permissions-manage.xhtml b/src/main/webapp/permissions-manage.xhtml index 819c6c59fcc..41ee8d02a3f 100644 --- a/src/main/webapp/permissions-manage.xhtml +++ b/src/main/webapp/permissions-manage.xhtml @@ -209,8 +209,9 @@
- #{bundle['dataverse.permissions.history.download']} diff --git a/src/test/java/edu/harvard/iq/dataverse/api/SignedUrlAuthMechanismIT.java b/src/test/java/edu/harvard/iq/dataverse/api/SignedUrlAuthMechanismIT.java index dd5ddbfa7d8..92556d87c2a 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/SignedUrlAuthMechanismIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/SignedUrlAuthMechanismIT.java @@ -6,6 +6,7 @@ import org.junit.jupiter.api.Test; import static io.restassured.RestAssured.get; +import static jakarta.ws.rs.core.Response.Status.BAD_REQUEST; import static jakarta.ws.rs.core.Response.Status.OK; import static jakarta.ws.rs.core.Response.Status.UNAUTHORIZED; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -42,5 +43,11 @@ public void testSignedUrlAuthMechanism() { String invalidSignedUrlPath = String.format("/api/v1/datasets/:persistentId/?persistentId=%s&until=2999-01-01T23:59:29.855&user=dataverseAdmin&method=GET&token=invalidToken", datasetPersistentId); Response invalidSignedUrlResponse = get(invalidSignedUrlPath); assertEquals(UNAUTHORIZED.getStatusCode(), invalidSignedUrlResponse.getStatusCode()); + + // A URL that already contains a reserved signing parameter is rejected with a 400 instead of + // being silently rewritten (through 6.10 the parameter was quietly stripped before signing). + String reservedParamPath = String.format("/api/v1/datasets/:persistentId/?persistentId=%s&token=abc", datasetPersistentId); + Response reservedParamResponse = UtilIT.createSignedUrl(apiToken, reservedParamPath, username); + assertEquals(BAD_REQUEST.getStatusCode(), reservedParamResponse.getStatusCode()); } } diff --git a/src/test/java/edu/harvard/iq/dataverse/api/auth/SignedUrlAuthMechanismTest.java b/src/test/java/edu/harvard/iq/dataverse/api/auth/SignedUrlAuthMechanismTest.java index 6fd7d2e1d8e..11bd500cb0a 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/auth/SignedUrlAuthMechanismTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/auth/SignedUrlAuthMechanismTest.java @@ -5,19 +5,33 @@ import edu.harvard.iq.dataverse.authorization.users.ApiToken; import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; import edu.harvard.iq.dataverse.authorization.users.User; +import edu.harvard.iq.dataverse.privateurl.PrivateUrlServiceBean; +import edu.harvard.iq.dataverse.settings.JvmSettings; +import edu.harvard.iq.dataverse.util.UrlSignerUtil; +import edu.harvard.iq.dataverse.util.testing.JvmSetting; +import edu.harvard.iq.dataverse.util.testing.LocalJvmSettings; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.mockito.Mockito; import jakarta.ws.rs.container.ContainerRequestContext; +import java.net.URLDecoder; +import java.nio.charset.StandardCharsets; +import java.util.List; + import static edu.harvard.iq.dataverse.api.auth.SignedUrlAuthMechanism.RESPONSE_MESSAGE_BAD_SIGNED_URL; import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +@LocalJvmSettings public class SignedUrlAuthMechanismTest { private static final String TEST_SIGNED_URL_TOKEN = "test-signed-url-token"; private static final String TEST_SIGNED_URL_USER_ID = "test-user"; + // A signing secret must be configured for signed-URL authentication to be accepted at all (the + // mechanism rejects signed URLs when none is set); the signing key is then this secret + the token. + private static final String TEST_SIGNING_SECRET = "test-signing-secret"; private SignedUrlAuthMechanism sut; @@ -30,7 +44,7 @@ public void setUp() { @Test public void testFindUserFromRequest_SignedUrlTokenNotProvided() throws WrappedAuthErrorResponse { - sut.authSvc = Mockito.mock(AuthenticationServiceBean.class); + sut.authSvc = mock(AuthenticationServiceBean.class); ContainerRequestContext testContainerRequest = new SignedUrlContainerRequestTestFake(null, null); User actual = sut.findUserFromRequest(testContainerRequest); @@ -39,12 +53,13 @@ public void testFindUserFromRequest_SignedUrlTokenNotProvided() throws WrappedAu } @Test + @JvmSetting(key = JvmSettings.API_SIGNING_SECRET, value = TEST_SIGNING_SECRET) public void testFindUserFromRequest_SignedUrlTokenProvided_UserExists_ValidSignedUrl_UserAuthenticated() throws WrappedAuthErrorResponse { - AuthenticationServiceBean authenticationServiceBeanStub = Mockito.mock(AuthenticationServiceBean.class); - Mockito.when(authenticationServiceBeanStub.getAuthenticatedUser(TEST_SIGNED_URL_USER_ID)).thenReturn(testAuthenticatedUser); - ApiToken apiTokenStub = Mockito.mock(ApiToken.class); - Mockito.when(apiTokenStub.getTokenString()).thenReturn(TEST_SIGNED_URL_TOKEN); - Mockito.when(authenticationServiceBeanStub.findApiTokenByUser(testAuthenticatedUser)).thenReturn(apiTokenStub); + AuthenticationServiceBean authenticationServiceBeanStub = mock(AuthenticationServiceBean.class); + when(authenticationServiceBeanStub.getAuthenticatedUser(TEST_SIGNED_URL_USER_ID)).thenReturn(testAuthenticatedUser); + ApiToken apiTokenStub = mock(ApiToken.class); + when(apiTokenStub.getTokenString()).thenReturn(TEST_SIGNED_URL_TOKEN); + when(authenticationServiceBeanStub.findApiTokenByUser(testAuthenticatedUser)).thenReturn(apiTokenStub); sut.authSvc = authenticationServiceBeanStub; @@ -55,12 +70,13 @@ public void testFindUserFromRequest_SignedUrlTokenProvided_UserExists_ValidSigne } @Test + @JvmSetting(key = JvmSettings.API_SIGNING_SECRET, value = TEST_SIGNING_SECRET) public void testFindUserFromRequest_SignedUrlTokenProvided_UserExists_InvalidSignedUrl_UserNotAuthenticated() { - AuthenticationServiceBean authenticationServiceBeanStub = Mockito.mock(AuthenticationServiceBean.class); - Mockito.when(authenticationServiceBeanStub.getAuthenticatedUser(TEST_SIGNED_URL_USER_ID)).thenReturn(testAuthenticatedUser); - ApiToken apiTokenStub = Mockito.mock(ApiToken.class); - Mockito.when(apiTokenStub.getTokenString()).thenReturn("different-token-from-the-signed-url"); - Mockito.when(authenticationServiceBeanStub.findApiTokenByUser(testAuthenticatedUser)).thenReturn(apiTokenStub); + AuthenticationServiceBean authenticationServiceBeanStub = mock(AuthenticationServiceBean.class); + when(authenticationServiceBeanStub.getAuthenticatedUser(TEST_SIGNED_URL_USER_ID)).thenReturn(testAuthenticatedUser); + ApiToken apiTokenStub = mock(ApiToken.class); + when(apiTokenStub.getTokenString()).thenReturn("different-token-from-the-signed-url"); + when(authenticationServiceBeanStub.findApiTokenByUser(testAuthenticatedUser)).thenReturn(apiTokenStub); sut.authSvc = authenticationServiceBeanStub; @@ -71,10 +87,11 @@ public void testFindUserFromRequest_SignedUrlTokenProvided_UserExists_InvalidSig } @Test + @JvmSetting(key = JvmSettings.API_SIGNING_SECRET, value = TEST_SIGNING_SECRET) public void testFindUserFromRequest_SignedUrlTokenProvided_UserExists_UserApiTokenDoesNotExist_UserNotAuthenticated() { - AuthenticationServiceBean authenticationServiceBeanStub = Mockito.mock(AuthenticationServiceBean.class); - Mockito.when(authenticationServiceBeanStub.getAuthenticatedUser(TEST_SIGNED_URL_USER_ID)).thenReturn(testAuthenticatedUser); - Mockito.when(authenticationServiceBeanStub.findApiTokenByUser(testAuthenticatedUser)).thenReturn(null); + AuthenticationServiceBean authenticationServiceBeanStub = mock(AuthenticationServiceBean.class); + when(authenticationServiceBeanStub.getAuthenticatedUser(TEST_SIGNED_URL_USER_ID)).thenReturn(testAuthenticatedUser); + when(authenticationServiceBeanStub.findApiTokenByUser(testAuthenticatedUser)).thenReturn(null); sut.authSvc = authenticationServiceBeanStub; @@ -85,9 +102,10 @@ public void testFindUserFromRequest_SignedUrlTokenProvided_UserExists_UserApiTok } @Test + @JvmSetting(key = JvmSettings.API_SIGNING_SECRET, value = TEST_SIGNING_SECRET) public void testFindUserFromRequest_SignedUrlTokenProvided_UserDoesNotExistForTheGivenId_UserNotAuthenticated() { - AuthenticationServiceBean authenticationServiceBeanStub = Mockito.mock(AuthenticationServiceBean.class); - Mockito.when(authenticationServiceBeanStub.getAuthenticatedUser(TEST_SIGNED_URL_USER_ID)).thenReturn(null); + AuthenticationServiceBean authenticationServiceBeanStub = mock(AuthenticationServiceBean.class); + when(authenticationServiceBeanStub.getAuthenticatedUser(TEST_SIGNED_URL_USER_ID)).thenReturn(null); sut.authSvc = authenticationServiceBeanStub; @@ -96,4 +114,184 @@ public void testFindUserFromRequest_SignedUrlTokenProvided_UserDoesNotExistForTh assertEquals(RESPONSE_MESSAGE_BAD_SIGNED_URL, wrappedUnauthorizedAuthErrorResponse.getMessage()); } + + // End-to-end validation through the REAL SignedUrlAuthMechanism (URLDecoder.decode + isValidUrl), + // which the isValidUrl-only tests in UrlSignerUtilTest do not exercise. These tests configure a + // signing secret (required for signed-URL auth to be accepted) and sign with signUrlWithApiKey, so + // the signing key is the secret + the API token - exactly as the mechanism reconstructs it. + + private void givenUserWithSigningKey(String key) { + AuthenticationServiceBean authStub = mock(AuthenticationServiceBean.class); + when(authStub.getAuthenticatedUser(TEST_SIGNED_URL_USER_ID)).thenReturn(testAuthenticatedUser); + ApiToken apiToken = mock(ApiToken.class); + when(apiToken.getTokenString()).thenReturn(key); + when(authStub.findApiTokenByUser(testAuthenticatedUser)).thenReturn(apiToken); + sut.authSvc = authStub; + } + + @Test + @JvmSetting(key = JvmSettings.API_SIGNING_SECRET, value = TEST_SIGNING_SECRET) + public void testEndToEnd_tamperedSignedUrl_userNotAuthenticated() { + givenUserWithSigningKey(TEST_SIGNED_URL_TOKEN); + String base = "http://localhost:8080/api/v1/datasets/:persistentId?persistentId=doi:10.5072/FK2/ABC"; + String signedUrl = UrlSignerUtil.signUrlWithApiKey(base, 1000, TEST_SIGNED_URL_USER_ID, "GET", TEST_SIGNED_URL_TOKEN); + // Alter the signed portion of the URL after signing -> the signature must no longer validate. + String tampered = signedUrl.replace("FK2/ABC", "FK2/HACKED"); + + ContainerRequestContext request = new SignedUrlContainerRequestTestFake(TEST_SIGNED_URL_TOKEN, TEST_SIGNED_URL_USER_ID, tampered); + + assertThrows(WrappedUnauthorizedAuthErrorResponse.class, () -> sut.findUserFromRequest(request)); + } + + @Test + public void testEndToEnd_noSigningSecret_signedUrlRejected() { + // With no signing secret configured, a URL signed with only the bare API token would still hash + // valid (key = "" + token) - but the mechanism must refuse it, otherwise a leaked/expired token + // or a guest key derived from the public URL could be used to forge a signed URL. + givenUserWithSigningKey(TEST_SIGNED_URL_TOKEN); + String base = "http://localhost:8080/api/v1/datasets/1"; + String signedUrl = UrlSignerUtil.signUrl(base, 1000, TEST_SIGNED_URL_USER_ID, "GET", TEST_SIGNED_URL_TOKEN); + + ContainerRequestContext request = new SignedUrlContainerRequestTestFake(TEST_SIGNED_URL_TOKEN, TEST_SIGNED_URL_USER_ID, signedUrl); + + assertThrows(WrappedUnauthorizedAuthErrorResponse.class, () -> sut.findUserFromRequest(request)); + } + + @Test + @JvmSetting(key = JvmSettings.API_SIGNING_SECRET, value = TEST_SIGNING_SECRET) + public void testFindUserFromRequest_malformedPrivateUrlUser_standard401Not500() { + // The user query param is attacker-controlled and reachable unauthenticated: a private-url + // user id with a non-numeric suffix, or one whose dataset has no private URL, must produce + // the standard 401 - not an unhandled NumberFormatException/NullPointerException 500. + sut.authSvc = mock(AuthenticationServiceBean.class); + sut.privateUrlSvc = mock(PrivateUrlServiceBean.class); // returns null for any dataset id + + ContainerRequestContext nonNumericSuffix = new SignedUrlContainerRequestTestFake(TEST_SIGNED_URL_TOKEN, "!abc"); + assertThrows(WrappedUnauthorizedAuthErrorResponse.class, () -> sut.findUserFromRequest(nonNumericSuffix)); + + ContainerRequestContext noPrivateUrlForDataset = new SignedUrlContainerRequestTestFake(TEST_SIGNED_URL_TOKEN, "!999999"); + assertThrows(WrappedUnauthorizedAuthErrorResponse.class, () -> sut.findUserFromRequest(noPrivateUrlForDataset)); + } + + // The primary signed-URL contract: whatever URL is submitted for signing - percent-escapes, '+' + // and all - the returned signed URL must authenticate when presented back VERBATIM. No client-side + // decoding, re-encoding or reconstruction may be required. + @Test + @JvmSetting(key = JvmSettings.API_SIGNING_SECRET, value = TEST_SIGNING_SECRET) + public void testEndToEnd_escapedUrlSignedAndUsedVerbatim_authenticates() { + givenUserWithSigningKey(TEST_SIGNED_URL_TOKEN); + List urls = List.of( + // url.QueryEscape'd persistentId, as rdm-integration's userPermissions/metadata calls send it + "http://localhost:8080/api/v1/datasets/:persistentId/userPermissions?persistentId=doi%3A10.5072%2FFK2%2FABC", + "http://localhost:8080/api/v1/datasets/:persistentId?persistentId=doi%3A10.5072%2FFK2%2FABC&excludeFiles=true", + // escaped search term: '+' for space, %3A / %22 for ':' and '"' + "http://localhost:8080/api/v1/mydata/retrieve?selected_page=1&mydata_search_term=text%3A%22hello+world%22", + // a value with an escaped literal '%' - decoding this twice would corrupt it + "http://localhost:8080/api/v1/search?q=100%2525done" + ); + for (String url : urls) { + String signedUrl = UrlSignerUtil.signUrlWithApiKey(url, 1000, TEST_SIGNED_URL_USER_ID, "GET", TEST_SIGNED_URL_TOKEN); + ContainerRequestContext request = new SignedUrlContainerRequestTestFake(TEST_SIGNED_URL_TOKEN, TEST_SIGNED_URL_USER_ID, signedUrl); + try { + assertEquals(testAuthenticatedUser, sut.findUserFromRequest(request), + "signed URL must authenticate when used verbatim: " + signedUrl); + } catch (WrappedAuthErrorResponse e) { + fail("signed URL must authenticate when used verbatim: " + signedUrl); + } + } + } + + @Test + @JvmSetting(key = JvmSettings.API_SIGNING_SECRET, value = TEST_SIGNING_SECRET) + public void testEndToEnd_tamperedEscapedUrlUsedVerbatim_rejected() { + givenUserWithSigningKey(TEST_SIGNED_URL_TOKEN); + String url = "http://localhost:8080/api/v1/datasets/:persistentId/userPermissions?persistentId=doi%3A10.5072%2FFK2%2FABC"; + String signedUrl = UrlSignerUtil.signUrlWithApiKey(url, 1000, TEST_SIGNED_URL_USER_ID, "GET", TEST_SIGNED_URL_TOKEN); + String tampered = signedUrl.replace("FK2%2FABC", "FK2%2FHACKED"); + + ContainerRequestContext request = new SignedUrlContainerRequestTestFake(TEST_SIGNED_URL_TOKEN, TEST_SIGNED_URL_USER_ID, tampered); + + assertThrows(WrappedUnauthorizedAuthErrorResponse.class, () -> sut.findUserFromRequest(request)); + } + + @Test + @JvmSetting(key = JvmSettings.API_SIGNING_SECRET, value = TEST_SIGNING_SECRET) + public void testEndToEnd_clientStackReEncodesSignatureParams_authenticates() { + // Why the decoded fallback must stay: every signed URL carries ':' in its own until timestamp + // (and clients/proxies like Apache HttpClient, OkHttp or mod_proxy may percent-encode it in + // flight). A URL signed in decoded form and presented in a re-encoded variant must validate. + givenUserWithSigningKey(TEST_SIGNED_URL_TOKEN); + String base = "http://localhost:8080/api/v1/datasets/42"; + String signedUrl = UrlSignerUtil.signUrlWithApiKey(base, 1000, TEST_SIGNED_URL_USER_ID, "GET", TEST_SIGNED_URL_TOKEN); + // Simulate a stack that re-encodes ':' in query values; the path is left alone. + int queryStart = signedUrl.indexOf('?'); + String reEncoded = signedUrl.substring(0, queryStart) + + signedUrl.substring(queryStart).replace(":", "%3A"); + + ContainerRequestContext request = new SignedUrlContainerRequestTestFake(TEST_SIGNED_URL_TOKEN, TEST_SIGNED_URL_USER_ID, reEncoded); + + try { + assertEquals(testAuthenticatedUser, sut.findUserFromRequest(request), + "re-encoded variant of a decoded-form-signed URL must still authenticate"); + } catch (WrappedAuthErrorResponse e) { + fail("re-encoded variant of a decoded-form-signed URL must still authenticate"); + } + } + + // Runs the real rdm flow: un-escape, sign, request the original (encoded) URL + signature, then the + // server URL-decodes the request and checks it. Returns true iff the user authenticates. + private boolean validatesEndToEndAsRdmClient(String urlAsClientBuilds) { + givenUserWithSigningKey(TEST_SIGNED_URL_TOKEN); + String canonical = URLDecoder.decode(urlAsClientBuilds, StandardCharsets.UTF_8); + String signed = UrlSignerUtil.signUrlWithApiKey(canonical, 1000, TEST_SIGNED_URL_USER_ID, "GET", TEST_SIGNED_URL_TOKEN); + String requestUri = urlAsClientBuilds + signed.substring(canonical.length()); + ContainerRequestContext request = new SignedUrlContainerRequestTestFake(TEST_SIGNED_URL_TOKEN, TEST_SIGNED_URL_USER_ID, requestUri); + try { + return testAuthenticatedUser.equals(sut.findUserFromRequest(request)); + } catch (WrappedAuthErrorResponse e) { + return false; + } + } + + @Test + @JvmSetting(key = JvmSettings.API_SIGNING_SECRET, value = TEST_SIGNING_SECRET) + public void testEndToEnd_allRdmIntegrationUrls_authenticate() { + final String s = "https://demo.dataverse.org"; + final String pid = "doi:10.5072/FK2/ABC"; // raw, as most rdm paths send it + final String escPid = "doi%3A10.5072%2FFK2%2FABC"; // url.QueryEscape form (GetDatasetMetadata, GetDatasetUserPermissions) + + // Every URL shape rdm-integration signs - each must authenticate end to end. + List urls = List.of( + // raw persistentId in the query (GetNodeMap, CheckPermission, globus, writes, dataverse plugin) + s + "/api/v1/datasets/:persistentId/versions/:latest/files?persistentId=" + pid, + s + "/api/v1/datasets/:persistentId?persistentId=" + pid, + s + "/api/v1/admin/permissions/:persistentId?persistentId=" + pid + "&unblock-key=UNBLOCK", + s + "/api/v1/datasets/:persistentId/requestGlobusUploadPaths?persistentId=" + pid, + s + "/api/v1/datasets/:persistentId/addGlobusFiles?persistentId=" + pid, + s + "/api/v1/datasets/:persistentId/requestGlobusDownload?persistentId=" + pid, + s + "/api/v1/datasets/:persistentId/monitorGlobusDownload?persistentId=" + pid, + s + "/api/v1/datasets/:persistentId/globusDownloadParameters?persistentId=" + pid + "&downloadId=globus-task-123", + s + "/api/v1/datasets/:persistentId/add?persistentId=" + pid, + s + "/api/v1/datasets/:persistentId/addFiles?persistentId=" + pid, + s + "/api/v1/datasets/:persistentId/replaceFiles?persistentId=" + pid, + s + "/api/v1/datasets/:persistentId/deleteFiles?persistentId=" + pid, + s + "/api/v1/datasets/:persistentId/cleanStorage?persistentId=" + pid, + // url-escaped persistentId (GetDatasetMetadata, GetDatasetUserPermissions) + s + "/api/v1/datasets/:persistentId?persistentId=" + escPid + "&excludeFiles=true", + s + "/api/v1/datasets/:persistentId/userPermissions?persistentId=" + escPid, + // mydata/retrieve: url-escaped search term, '+' for spaces, repeated query params + s + "/api/v1/mydata/retrieve?selected_page=1&dvobject_types=Dataset" + + "&published_states=Published&published_states=Unpublished&published_states=Draft" + + "&role_ids=1&role_ids=6&mydata_search_term=text%3A%22hello+world%22", + // numeric-id / no-persistentId paths + s + "/api/v1/access/datafile/123/metadata/ddi", + s + "/api/v1/access/datafile/123", + s + "/api/v1/files/123", + s + "/api/v1/users/:me", + s + "/api/v1/datasets/42/versions/:latest?excludeFiles=true" + ); + for (String url : urls) { + assertTrue(validatesEndToEndAsRdmClient(url), "signed URL must authenticate end to end: " + url); + } + } } diff --git a/src/test/java/edu/harvard/iq/dataverse/api/auth/doubles/SignedUrlContainerRequestTestFake.java b/src/test/java/edu/harvard/iq/dataverse/api/auth/doubles/SignedUrlContainerRequestTestFake.java index df37f6723d3..c4c1360f2fc 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/auth/doubles/SignedUrlContainerRequestTestFake.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/auth/doubles/SignedUrlContainerRequestTestFake.java @@ -6,11 +6,23 @@ public class SignedUrlContainerRequestTestFake extends ContainerRequestTestFake private final UriInfo uriInfo; public SignedUrlContainerRequestTestFake(String signedUrlToken, String signedUrlUserId) { - this.uriInfo = new SignedUrlUriInfoTestFake(signedUrlToken, signedUrlUserId); + this(signedUrlToken, signedUrlUserId, null); + } + + public SignedUrlContainerRequestTestFake(String signedUrlToken, String signedUrlUserId, String requestUriOverride) { + this.uriInfo = new SignedUrlUriInfoTestFake(signedUrlToken, signedUrlUserId, requestUriOverride); } @Override public UriInfo getUriInfo() { return uriInfo; } + + @Override + public String getMethod() { + // The signed URLs these fakes build are signed for GET; returning it (rather than the base + // class's null) makes SignedUrlAuthMechanism actually exercise the HTTP-method check in + // UrlSignerUtil.isValidUrl instead of silently skipping it. + return "GET"; + } } diff --git a/src/test/java/edu/harvard/iq/dataverse/api/auth/doubles/SignedUrlUriInfoTestFake.java b/src/test/java/edu/harvard/iq/dataverse/api/auth/doubles/SignedUrlUriInfoTestFake.java index fa9da7fc8de..4c3e5a33101 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/auth/doubles/SignedUrlUriInfoTestFake.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/auth/doubles/SignedUrlUriInfoTestFake.java @@ -6,6 +6,8 @@ import jakarta.ws.rs.core.MultivaluedMap; import java.net.URI; +import java.net.URLDecoder; +import java.nio.charset.StandardCharsets; import static edu.harvard.iq.dataverse.util.UrlSignerUtil.SIGNED_URL_TOKEN; import static edu.harvard.iq.dataverse.util.UrlSignerUtil.SIGNED_URL_USER; @@ -15,24 +17,51 @@ public class SignedUrlUriInfoTestFake extends UriInfoTestFake { private final String signedUrlToken; private final String signedUrlUserId; + private final String requestUriOverride; private static final String SIGNED_URL_BASE_URL = "http://localhost:8080/api/test1"; private static final Integer SIGNED_URL_TIMEOUT = 1000; public SignedUrlUriInfoTestFake(String signedUrlToken, String signedUrlUserId) { + this(signedUrlToken, signedUrlUserId, null); + } + + // Lets a test supply the exact request URI the server would see, to exercise the real + // URLDecoder.decode + isValidUrl validation path. + public SignedUrlUriInfoTestFake(String signedUrlToken, String signedUrlUserId, String requestUriOverride) { this.signedUrlToken = signedUrlToken; this.signedUrlUserId = signedUrlUserId; + this.requestUriOverride = requestUriOverride; } @Override public URI getRequestUri() { - return URI.create(UrlSignerUtil.signUrl(SIGNED_URL_BASE_URL, SIGNED_URL_TIMEOUT, signedUrlUserId, GET, signedUrlToken)); + if (requestUriOverride != null) { + return URI.create(requestUriOverride); + } + // Sign the way the server does: with the configured signing secret prepended to the token. This + // keeps the fake consistent with SignedUrlAuthMechanism, which validates with secret + token. + return URI.create(UrlSignerUtil.signUrlWithApiKey(SIGNED_URL_BASE_URL, SIGNED_URL_TIMEOUT, signedUrlUserId, GET, signedUrlToken)); } @Override public MultivaluedMap getQueryParameters() { MultivaluedMap queryParameters = new MultivaluedHashMap<>(); + if (requestUriOverride != null) { + // Reflect the actual request URI (like Jersey would) so tests exercise real extraction of + // the user/token params from the URL under test, not fabricated values. + String query = URI.create(requestUriOverride).getRawQuery(); + if (query != null) { + for (String pair : query.split("&")) { + int equals = pair.indexOf('='); + String name = (equals < 0) ? pair : pair.substring(0, equals); + String value = (equals < 0) ? "" : URLDecoder.decode(pair.substring(equals + 1), StandardCharsets.UTF_8); + queryParameters.add(name, value); + } + } + return queryParameters; + } queryParameters.add(SIGNED_URL_TOKEN, signedUrlToken); queryParameters.add(SIGNED_URL_USER, signedUrlUserId); return queryParameters; diff --git a/src/test/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandlerTest.java b/src/test/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandlerTest.java index 0d4d8e3a4a2..1adfbf615cb 100644 --- a/src/test/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandlerTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandlerTest.java @@ -213,6 +213,7 @@ public void testGetToolUrlWithOptionalQueryParameters() { @Test @JvmSetting(key = JvmSettings.SITE_URL, value = "https://librascholar.org") + @JvmSetting(key = JvmSettings.API_SIGNING_SECRET, value = "test-only-signing-secret") public void testGetToolUrlWithAllowedApiCalls() { System.out.println("allowedApiCalls test"); Dataset ds = new Dataset(); @@ -241,6 +242,81 @@ public void testGetToolUrlWithAllowedApiCalls() { System.out.println(JsonUtil.prettyPrint(jo)); } + @Test + @JvmSetting(key = JvmSettings.SITE_URL, value = "https://librascholar.org") + @JvmSetting(key = JvmSettings.API_SIGNING_SECRET, value = "test-only-signing-secret") + public void testGetToolUrlWithAllowedApiCallsStripsReservedParameters() { + // A manifest can (mis)use reserved words in an allowedApiCalls urlTemplate - most dangerously + // key={apiToken}. The user's real API token must never end up in the URL handed to the tool, + // and stray reserved params must not spoof or break the signing. + Dataset ds = new Dataset(); + ds.setId(1L); + ApiToken at = new ApiToken(); + AuthenticatedUser au = new AuthenticatedUser(); + au.setUserIdentifier("dataverseAdmin"); + at.setAuthenticatedUser(au); + at.setTokenString("secret-api-token-1234"); + ExternalTool et = getToolWithAllowedApiCallsUrlTemplate("/api/v1/datasets/{datasetId}?key={apiToken}&signed=true&user=Fred"); + URLTokenUtil handler = new ExternalToolHandler(et, ds, at, null); + JsonObject jo = handler + .createPostBody(handler.getParams(JsonUtil.getJsonObject(et.getToolParameters())), JsonUtil.getJsonArray(et.getAllowedApiCalls())).build(); + String signedUrl = jo.getJsonArray("signedUrls").getJsonObject(0).getString("signedUrl"); + assertFalse(signedUrl.contains("secret-api-token-1234"), "the user's API token must not appear in the signed URL"); + assertFalse(signedUrl.contains("key=")); + assertFalse(signedUrl.contains("signed=true")); + assertFalse(signedUrl.contains("user=Fred")); + assertTrue(signedUrl.contains("https://librascholar.org/api/v1/datasets/1")); + assertTrue(signedUrl.contains("user=dataverseAdmin")); + assertTrue(signedUrl.contains("&token=")); + } + + private static ExternalTool getToolWithAllowedApiCallsUrlTemplate(String urlTemplate) { + String tool = JsonUtil.createObjectBuilder() + .add("displayName", "AwesomeTool") + .add("toolName", "explorer") + .add("description", "This tool is awesome.") + .add("types", JsonUtil.createArrayBuilder().add("explore")) + .add("scope", "dataset") + .add("toolUrl", "http://awesometool.com") + .add("hasPreviewMode", "true") + .add("toolParameters", JsonUtil.createObjectBuilder() + .add("httpMethod", "GET") + .add("queryParameters", JsonUtil.createArrayBuilder() + .add(JsonUtil.createObjectBuilder().add("datasetId", "{datasetId}")))) + .add("allowedApiCalls", JsonUtil.createArrayBuilder() + .add(JsonUtil.createObjectBuilder() + .add("name", "getDataset") + .add("httpMethod", "GET") + .add("urlTemplate", urlTemplate) + .add("timeOut", 10))) + .build().toString(); + return ExternalToolServiceBean.parseAddExternalToolManifest(tool); + } + + @Test + @JvmSetting(key = JvmSettings.SITE_URL, value = "https://librascholar.org") + public void testGetToolUrlWithAllowedApiCallsNoSigningSecret() { + // Without dataverse.api.signing-secret configured, the URL must be sent unsigned (no signing + // parameters at all) instead of weakly signed - and no IllegalStateException may escape. + Dataset ds = new Dataset(); + ds.setId(1L); + ApiToken at = new ApiToken(); + AuthenticatedUser au = new AuthenticatedUser(); + au.setUserIdentifier("dataverseAdmin"); + at.setAuthenticatedUser(au); + at.setTokenString("1234"); + ExternalTool et = ExternalToolServiceBeanTest.getAllowedApiCallsTool(); + URLTokenUtil externalToolHandler = new ExternalToolHandler(et, ds, at, null); + JsonObject jo = externalToolHandler + .createPostBody(externalToolHandler.getParams(JsonUtil.getJsonObject(et.getToolParameters())), JsonUtil.getJsonArray(et.getAllowedApiCalls())).build(); + String signedUrl = jo.getJsonArray("signedUrls").getJsonObject(0).getString("signedUrl"); + assertEquals("https://librascholar.org/api/v1/datasets/1", signedUrl); + assertFalse(signedUrl.contains("until=")); + assertFalse(signedUrl.contains("user=")); + assertFalse(signedUrl.contains("method=")); + assertFalse(signedUrl.contains("token=")); + } + @Test @JvmSetting(key = JvmSettings.SITE_URL, value = "https://librascholar.org") public void testDatasetConfigureTool() { diff --git a/src/test/java/edu/harvard/iq/dataverse/util/UrlSignerUtilTest.java b/src/test/java/edu/harvard/iq/dataverse/util/UrlSignerUtilTest.java index d92f8822e59..aac8906322e 100644 --- a/src/test/java/edu/harvard/iq/dataverse/util/UrlSignerUtilTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/util/UrlSignerUtilTest.java @@ -1,16 +1,20 @@ package edu.harvard.iq.dataverse.util; -import jakarta.ws.rs.core.MultivaluedHashMap; -import jakarta.ws.rs.core.MultivaluedMap; +import edu.harvard.iq.dataverse.settings.JvmSettings; +import edu.harvard.iq.dataverse.util.testing.JvmSetting; +import edu.harvard.iq.dataverse.util.testing.LocalJvmSettings; import org.junit.jupiter.api.Test; -import java.util.List; import java.util.logging.Level; import java.util.logging.Logger; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +@LocalJvmSettings public class UrlSignerUtilTest { @Test @@ -54,36 +58,220 @@ public void testSignAndValidate() { @Test public void testSignAndValidateWithParams() { final String url1 = "http://localhost:8080/api/test1?p1=true&p2=test"; - final String url2 = "http://localhost:8080/api/test1?p1=true&p2=test&until=2999-01-01&user=Fred&method=POST&token=abracadabara&signed=true"; - final String url3 = "localhost:8080/api/test1?p1=true&p2&until=2099-01-01"; + final String url3 = "localhost:8080/api/test1?p1=true&p2&p3=1"; final int longTimeout = 1000; final String user1 = "Alice"; final String key = "abracadabara open sesame"; - MultivaluedMap queryParameters = new MultivaluedHashMap<>(); - queryParameters.put("p1", List.of("true")); - queryParameters.put("p2", List.of("test")); - queryParameters.put("until", List.of("2099-01-01")); String signedUrl1 = UrlSignerUtil.signUrl(url1, longTimeout, user1, "GET", key); assertTrue(signedUrl1.contains("test1?p1=true&p2=test")); - System.out.println(signedUrl1); - - String signedUrl2 = UrlSignerUtil.signUrl(url2, longTimeout, user1, "GET", key); - assertTrue(signedUrl2.contains("&until=")); // contains the until param but not the bogus one passed in - assertFalse(signedUrl2.contains("&until=2099-01-01")); - assertTrue(signedUrl2.contains("&user=Alice")); // contains the user param but not the bogus one passed in - assertFalse(signedUrl2.contains("&user=Fred")); - assertTrue(signedUrl2.contains("&method=GET")); // contains the method param but not the bogus one passed in - assertFalse(signedUrl2.contains("&method=POST")); - assertTrue(signedUrl2.contains("&token=")); // contains the signed token param but not the bogus one passed in - assertFalse(signedUrl2.contains("&token=abracadabara")); - assertFalse(signedUrl2.contains("&signed")); // make sure we don't propagate the "signed" param - System.out.println(signedUrl2); - - // This will log an error but will still return the signed url even if it's now a valid url - // All callers of this method don't handle errors being returned, and it's highly unlikely that the url would be bad + assertTrue(UrlSignerUtil.isValidUrl(signedUrl1, user1, "GET", key)); + + // Works with a non-absolute URL and with params that have no value. String signedUrl3 = UrlSignerUtil.signUrl(url3, longTimeout, user1, "GET", key); - System.out.println(signedUrl3); - assertTrue(signedUrl3.contains("&p2&")); // Show that this works with params that have no value + assertTrue(signedUrl3.contains("&p2&")); + } + + @Test + public void testSignUrlThrowsOnReservedSigningParameters() { + final int longTimeout = 1000; + final String user1 = "Alice"; + final String key = "abracadabara open sesame"; + + // A base URL that already contains a param the algorithm appends is a caller bug: signUrl + // must refuse it instead of signing a different URL than the caller intended. + String[] badUrls = new String[] { + "http://localhost:8080/api/test1?p1=true&until=2999-01-01", + "http://localhost:8080/api/test1?user=Fred", + "http://localhost:8080/api/test1?p1=true&method=POST&p2=test", + "http://localhost:8080/api/test1?p1=true&token=abracadabara", + "http://localhost:8080/api/test1?until", // reserved name without a value + "http://localhost:8080/api/test1?p1=true&token=abracadabara#frag", + }; + for (String badUrl : badUrls) { + assertThrows(IllegalArgumentException.class, + () -> UrlSignerUtil.signUrl(badUrl, longTimeout, user1, "GET", key), + "signUrl must reject a base URL already containing a signing param: " + badUrl); + } + + // Names that merely contain a reserved name are fine, as is a reserved name inside the + // fragment, which is not part of the query. + UrlSignerUtil.signUrl("http://localhost:8080/api/test1?tokens=1&xtoken=2&user2=3", longTimeout, user1, "GET", key); + UrlSignerUtil.signUrl("http://localhost:8080/api/test1#frag?token=1", longTimeout, user1, "GET", key); + + // The Dataverse request-level params "signed" and "key" are NOT the utility's concern: + // signUrl leaves them untouched, byte for byte. Callers that must not sign them strip them + // (Access) or reject the URL (requestSignedUrl) before calling signUrl. + String withSignedAndKey = "http://localhost:8080/api/test1?p1=true&signed=true&key=abc"; + String signedUrl = UrlSignerUtil.signUrl(withSignedAndKey, longTimeout, user1, "GET", key); + assertTrue(signedUrl.startsWith(withSignedAndKey + "&")); + assertTrue(UrlSignerUtil.isValidUrl(signedUrl, user1, "GET", key)); + } + + @Test + public void testFindReservedParameter() { + assertEquals("token", UrlSignerUtil.findReservedParameter( + "http://x/api?a=1&token=y", UrlSignerUtil.signingParameters)); + assertEquals("until", UrlSignerUtil.findReservedParameter( + "http://x/api?until", UrlSignerUtil.signingParameters)); + assertNull(UrlSignerUtil.findReservedParameter( + "http://x/api?signed=true&key=abc", UrlSignerUtil.signingParameters)); + assertEquals("signed", UrlSignerUtil.findReservedParameter( + "http://x/api?signed=true&key=abc", UrlSignerUtil.reservedParameters)); + assertNull(UrlSignerUtil.findReservedParameter( + "http://x/api?tokens=1&xtoken=2&user2=3", UrlSignerUtil.reservedParameters)); + assertNull(UrlSignerUtil.findReservedParameter( + "http://x/api", UrlSignerUtil.reservedParameters)); + assertNull(UrlSignerUtil.findReservedParameter( + "http://x/api#frag?token=1", UrlSignerUtil.reservedParameters)); + } + + @Test + public void testStripReservedParametersPreservesSpecialCharacters() { + // The signature is a byte-exact MAC over the URL string, so removing the reserved signing + // params must not re-encode anything else: a DOI's ':' and '/' must survive unchanged. + String doiUrl = "http://localhost:8080/api/v1/datasets/:persistentId?persistentId=doi:10.5072/FK2/ABC123&foo=bar"; + assertEquals(doiUrl, UrlSignerUtil.stripReservedParameters(doiUrl)); + + // Reserved params (here token/user/signed) are removed; everything else is left byte-for-byte. + String withReserved = "http://localhost:8080/api/v1/datasets/:persistentId?persistentId=doi:10.5072/FK2/ABC123&token=spoofed&user=Mallory&signed=true&foo=bar"; + assertEquals("http://localhost:8080/api/v1/datasets/:persistentId?persistentId=doi:10.5072/FK2/ABC123&foo=bar", + UrlSignerUtil.stripReservedParameters(withReserved)); + + // A URL with no query string is returned unchanged. + String noQuery = "http://localhost:8080/api/v1/datasets/:persistentId"; + assertEquals(noQuery, UrlSignerUtil.stripReservedParameters(noQuery)); + } + + @Test + public void testSignAndValidateSpecialCharacters() { + final int longTimeout = 1000; + final String user = "Alice"; + final String method = "GET"; + final String key = "abracadabara open sesame"; + + // DOIs with ':' and '/', pre-encoded values, spaces, unicode and embedded URLs must all sign + // byte-exact and validate over those exact bytes. End-to-end validation including the + // server-side URL decoding is covered in SignedUrlAuthMechanismTest. + String[] baseUrls = new String[] { + "http://localhost:8080/api/v1/datasets/:persistentId?persistentId=doi:10.5072/FK2/ABC123&foo=bar", + "http://localhost:8080/api/v1/datasets/:persistentId?persistentId=doi%3A10.5072%2FFK2%2FABC123", + "http://localhost:8080/api/v1/search?q=hello%20world&persistentId=doi:10.1/2", + "http://localhost:8080/api/v1/search?q=hello world&pid=doi:10.1/2", + "http://localhost:8080/api/v1/search?q=café&name=測試", + "http://localhost:8080/api/v1/redirect?url=http%3A%2F%2Fexample.com%2Ff%3Fa%3D1%26b%3D2" + }; + for (String baseUrl : baseUrls) { + String signedUrl = UrlSignerUtil.signUrl(baseUrl, longTimeout, user, method, key); + // The base URL is preserved byte-for-byte in the signed URL (no re-encoding). + assertTrue(signedUrl.startsWith(baseUrl + "&"), + "base URL must be preserved byte-for-byte: " + signedUrl); + assertTrue(UrlSignerUtil.isValidUrl(signedUrl, user, method, key), + "signed URL should validate when used verbatim: " + signedUrl); + } + } + + @Test + public void testSignedUrlIsByteExact() { + // Byte-exact contract: the signature is over the URL as provided, so a re-encoded variant + // must fail validation. This is the regression that URIBuilder normalization caused. + final int longTimeout = 1000; + final String user = "Alice"; + final String method = "GET"; + final String key = "abracadabara open sesame"; + + String baseUrl = "http://localhost:8080/api/v1/datasets/:persistentId?persistentId=doi:10.5072/FK2/ABC123"; + String signedUrl = UrlSignerUtil.signUrl(baseUrl, longTimeout, user, method, key); + assertTrue(UrlSignerUtil.isValidUrl(signedUrl, user, method, key)); + + // Re-encoding ':' and '/' in the DOI changes the signed bytes, so it must be rejected. + String reEncoded = signedUrl.replace("doi:10.5072/FK2/ABC123", "doi%3A10.5072%2FFK2%2FABC123"); + assertFalse(UrlSignerUtil.isValidUrl(reEncoded, user, method, key), + "a re-encoded variant must not validate (byte-exact contract)"); + } + + @Test + public void testStripReservedParametersEdgeCases() { + // Empty query segments are part of the byte-exact contract and must survive unchanged. + assertEquals("http://x/api?&a=b", UrlSignerUtil.stripReservedParameters("http://x/api?&a=b")); + assertEquals("http://x/api?a=1&", UrlSignerUtil.stripReservedParameters("http://x/api?a=1&")); + assertEquals("http://x/api?a=1&&b=2", UrlSignerUtil.stripReservedParameters("http://x/api?a=1&&b=2")); + assertEquals("http://x/api?a=b&&", UrlSignerUtil.stripReservedParameters("http://x/api?a=b&&")); + assertEquals("http://x/api?", UrlSignerUtil.stripReservedParameters("http://x/api?")); + + // A reserved name with no '=' is still stripped. + assertEquals("http://x/api", UrlSignerUtil.stripReservedParameters("http://x/api?until")); + assertEquals("http://x/api?a=1", UrlSignerUtil.stripReservedParameters("http://x/api?until&a=1")); + + // Fragments survive byte-for-byte, even when attached to a stripped reserved parameter. + assertEquals("http://x/api?a=1#frag", UrlSignerUtil.stripReservedParameters("http://x/api?a=1#frag")); + assertEquals("http://x/api?a=1#frag", UrlSignerUtil.stripReservedParameters("http://x/api?a=1&token=y#frag")); + assertEquals("http://x/api#frag", UrlSignerUtil.stripReservedParameters("http://x/api?token=y#frag")); + + // A '?' inside the fragment is not a query: nothing to strip, returned unchanged. + assertEquals("http://x/api#frag?until=1", UrlSignerUtil.stripReservedParameters("http://x/api#frag?until=1")); + + // Non-reserved names that merely contain a reserved name are kept. + assertEquals("http://x/api?tokens=1&xtoken=2&user2=3", + UrlSignerUtil.stripReservedParameters("http://x/api?tokens=1&xtoken=2&user2=3")); + } + + @Test + public void testSignAndValidateEmptyQuerySegments() { + // Degenerate-but-legal query shapes must round-trip byte-exactly through sign + validate, + // so suffix-reconstructing clients (signed.substring(base.length())) keep working. + final String user = "Alice"; + final String method = "GET"; + final String key = "abracadabara open sesame"; + String[] baseUrls = new String[] { + "http://localhost:8080/api/v1/x?&a=b", + "http://localhost:8080/api/v1/x?a=1&", + "http://localhost:8080/api/v1/x?a=1&&b=2", + }; + for (String baseUrl : baseUrls) { + String signedUrl = UrlSignerUtil.signUrl(baseUrl, 1000, user, method, key); + assertTrue(signedUrl.startsWith(baseUrl + "&"), + "base URL must be preserved byte-for-byte: " + signedUrl); + assertTrue(UrlSignerUtil.isValidUrl(signedUrl, user, method, key), + "signed URL should validate when used verbatim: " + signedUrl); + } + } + + @Test + public void testIsSigningSecretConfiguredWithoutSecret() { + assertFalse(UrlSignerUtil.isSigningSecretConfigured()); + } + + @Test + @JvmSetting(key = JvmSettings.API_SIGNING_SECRET, value = "test-only-signing-secret") + public void testIsSigningSecretConfiguredWithSecret() { + assertTrue(UrlSignerUtil.isSigningSecretConfigured()); + } + + @Test + public void testSignUrlWithApiKeyRequiresSecret() { + // Without a signing secret the API-token-based signing entry point must refuse to produce + // a weakly-keyed URL. + assertThrows(IllegalStateException.class, + () -> UrlSignerUtil.signUrlWithApiKey("http://localhost:8080/api/test1", 1000, "Alice", "GET", "some-api-token")); + } + + @Test + @JvmSetting(key = JvmSettings.API_SIGNING_SECRET, value = "test-only-signing-secret") + public void testSignUrlWithApiKeySignsWithSecretPrependedToApiKey() { + final String baseUrl = "http://localhost:8080/api/v1/datasets/:persistentId?persistentId=doi:10.5072/FK2/ABC123"; + final String user = "Alice"; + final String method = "GET"; + final String apiKey = "some-api-token"; + + String signedUrl = UrlSignerUtil.signUrlWithApiKey(baseUrl, 1000, user, method, apiKey); + + // SignedUrlAuthMechanism reconstructs the key as + ; the + // signature must validate against exactly that combination and nothing weaker. + assertTrue(UrlSignerUtil.isValidUrl(signedUrl, user, method, "test-only-signing-secret" + apiKey)); + assertFalse(UrlSignerUtil.isValidUrl(signedUrl, user, method, apiKey), + "the API token alone must not validate a URL signed with the secret"); + assertFalse(UrlSignerUtil.isValidUrl(signedUrl, user, method, "test-only-signing-secret"), + "the secret alone must not validate a URL signed with secret+token"); } }