diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCLiveStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCLiveStreamExtractor.java index ba362f0bc7..80229bde76 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCLiveStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCLiveStreamExtractor.java @@ -226,10 +226,7 @@ private static final class MediaCCCLiveStreamMapperDTO { private List getStreams( @Nonnull final String streamType, @Nonnull final Function converter) { - return room.getArray(STREAMS).stream() - // Ensure that we use only process JsonObjects - .filter(JsonObject.class::isInstance) - .map(JsonObject.class::cast) + return room.getArray(STREAMS).streamAsJsonObjects() // Only process streams of requested type .filter(streamJsonObj -> streamType.equals(streamJsonObj.getString("type"))) // Flatmap Urls and ensure that we use only process JsonObjects diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelHelper.java index fe4f64f0f4..f59d775514 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelHelper.java @@ -1,10 +1,12 @@ package org.schabi.newpipe.extractor.services.youtube; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.STRING_PREDICATE; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.defaultAlertsCheck; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.hasArtistOrVerifiedIconBadgeAttachment; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.TITLE; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; import com.grack.nanojson.JsonObject; @@ -19,6 +21,7 @@ import java.io.IOException; import java.io.Serializable; import java.nio.charset.StandardCharsets; +import java.util.Objects; import java.util.Optional; import javax.annotation.Nonnull; @@ -38,7 +41,6 @@ public final class YoutubeChannelHelper { private static final String HEADER = "header"; private static final String PAGE_HEADER_VIEW_MODEL = "pageHeaderViewModel"; private static final String TAB_RENDERER = "tabRenderer"; - private static final String TITLE = "title"; private static final String TOPIC_CHANNEL_DETAILS_RENDERER = "topicChannelDetailsRenderer"; private YoutubeChannelHelper() { @@ -441,56 +443,44 @@ public static String getChannelId( @Nullable final ChannelHeader channelHeader, @Nonnull final JsonObject jsonResponse, @Nullable final String fallbackChannelId) throws ParsingException { - if (channelHeader != null) { - switch (channelHeader.headerType) { - case C4_TABBED: - final String channelId = channelHeader.json.getObject(HEADER) - .getObject(C4_TABBED_HEADER_RENDERER) - .getString("channelId", ""); - if (!isNullOrEmpty(channelId)) { - return channelId; - } - final String navigationC4TabChannelId = channelHeader.json - .getObject("navigationEndpoint") - .getObject(BROWSE_ENDPOINT) - .getString(BROWSE_ID); - if (!isNullOrEmpty(navigationC4TabChannelId)) { - return navigationC4TabChannelId; - } - break; - case CAROUSEL: - final String navigationCarouselChannelId = channelHeader.json.getObject(HEADER) - .getObject(CAROUSEL_HEADER_RENDERER) - .getArray(CONTENTS) - .streamAsJsonObjects() - .filter(item -> item.has(TOPIC_CHANNEL_DETAILS_RENDERER)) - .findFirst() - .orElse(new JsonObject()) - .getObject(TOPIC_CHANNEL_DETAILS_RENDERER) - .getObject("navigationEndpoint") - .getObject(BROWSE_ENDPOINT) - .getString(BROWSE_ID); - if (!isNullOrEmpty(navigationCarouselChannelId)) { - return navigationCarouselChannelId; - } - break; - default: - break; - } + final var headerType = channelHeader != null ? channelHeader.headerType : null; + final Optional channelIdOptional; + if (headerType == ChannelHeader.HeaderType.C4_TABBED) { + final String channelId = channelHeader.json.getObject(HEADER) + .getObject(C4_TABBED_HEADER_RENDERER) + .getString("channelId"); + channelIdOptional = Optional.ofNullable(channelId) + .or(() -> Optional.ofNullable(getBrowseId(channelHeader.json))); + } else if (headerType == ChannelHeader.HeaderType.CAROUSEL) { + channelIdOptional = channelHeader.json.getObject(HEADER) + .getObject(CAROUSEL_HEADER_RENDERER) + .getArray(CONTENTS) + .streamAsJsonObjects() + .map(item -> item.getObject(TOPIC_CHANNEL_DETAILS_RENDERER, null)) + .filter(Objects::nonNull) + .findFirst() + .map(YoutubeChannelHelper::getBrowseId); + } else { + channelIdOptional = Optional.empty(); } - final String externalChannelId = jsonResponse.getObject("metadata") - .getObject("channelMetadataRenderer") - .getString("externalChannelId"); - if (!isNullOrEmpty(externalChannelId)) { - return externalChannelId; - } + return channelIdOptional + .or(() -> { + final String externalChannelId = jsonResponse.getObject("metadata") + .getObject("channelMetadataRenderer") + .getString("externalChannelId"); + return Optional.ofNullable(externalChannelId); + }) + .or(() -> Optional.ofNullable(fallbackChannelId)) + .filter(STRING_PREDICATE) + .orElseThrow(() -> new ParsingException("Could not get channel ID")); + } - if (!isNullOrEmpty(fallbackChannelId)) { - return fallbackChannelId; - } else { - throw new ParsingException("Could not get channel ID"); - } + @Nullable + private static String getBrowseId(@Nonnull final JsonObject jsonObject) { + return jsonObject.getObject("navigationEndpoint") + .getObject(BROWSE_ENDPOINT) + .getString(BROWSE_ID); } @Nonnull @@ -498,40 +488,34 @@ public static String getChannelName(@Nullable final ChannelHeader channelHeader, @Nullable final JsonObject channelAgeGateRenderer, @Nonnull final JsonObject jsonResponse) throws ParsingException { - if (channelAgeGateRenderer != null) { - final String title = channelAgeGateRenderer.getString("channelTitle"); - if (isNullOrEmpty(title)) { - throw new ParsingException("Could not get channel name"); - } - return title; - } - - final String metadataRendererTitle = jsonResponse.getObject("metadata") - .getObject("channelMetadataRenderer") - .getString(TITLE); - if (!isNullOrEmpty(metadataRendererTitle)) { - return metadataRendererTitle; - } - - return Optional.ofNullable(channelHeader) - .map(header -> { - final JsonObject channelJson = header.json; - switch (header.headerType) { - case PAGE: - return channelJson.getObject(CONTENT) - .getObject(PAGE_HEADER_VIEW_MODEL) - .getObject(TITLE) - .getObject("dynamicTextViewModel") - .getObject("text") - .getString(CONTENT, channelJson.getString("pageTitle")); - case CAROUSEL: - case INTERACTIVE_TABBED: - return getTextFromObject(channelJson.getObject(TITLE)); - case C4_TABBED: - default: - return channelJson.getString(TITLE); - } - }) + final String channelTitle = channelAgeGateRenderer != null + ? channelAgeGateRenderer.getString("channelTitle") : null; + return Optional.ofNullable(channelTitle) + .or(() -> Optional.ofNullable(jsonResponse.getObject("metadata") + .getObject("channelMetadataRenderer") + .getString(TITLE))) + .filter(STRING_PREDICATE) + .or(() -> Optional.ofNullable(channelHeader) + .flatMap(header -> { + final JsonObject channelJson = header.json; + switch (header.headerType) { + case PAGE: + final String pageTitle = channelJson.getObject(CONTENT) + .getObject(PAGE_HEADER_VIEW_MODEL) + .getObject(TITLE) + .getObject("dynamicTextViewModel") + .getObject("text") + .getString(CONTENT, channelJson.getString("pageTitle")); + return Optional.ofNullable(pageTitle); + case CAROUSEL: + case INTERACTIVE_TABBED: + return getTextFromObject(channelJson.getObject(TITLE)); + case C4_TABBED: + default: + return Optional.ofNullable(channelJson.getString(TITLE)); + } + }) + ) // The channel name from a microformatDataRenderer may be different from the one // displayed, especially for auto-generated channels, depending on the language // requested for the interface (hl parameter of InnerTube requests' payload) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeDescriptionHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeDescriptionHelper.java index 0b3bf13bc8..4ce1e72176 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeDescriptionHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeDescriptionHelper.java @@ -239,7 +239,8 @@ private static void addAllCommandRuns( return; } - final String url = getUrlFromNavigationEndpoint(navigationEndpoint); + final String url = getUrlFromNavigationEndpoint(navigationEndpoint) + .orElse(null); if (url == null) { return; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMetaInfoHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMetaInfoHelper.java index bc8d9f85d0..c0d850fd62 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMetaInfoHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMetaInfoHelper.java @@ -1,20 +1,12 @@ package org.schabi.newpipe.extractor.services.youtube; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.extractCachedUrlIfNeeded; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObjectOrThrow; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isGoogleURL; -import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; -import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps; - import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; - import org.schabi.newpipe.extractor.MetaInfo; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.stream.Description; +import javax.annotation.Nonnull; import java.net.MalformedURLException; import java.net.URL; import java.util.ArrayList; @@ -23,14 +15,19 @@ import java.util.function.Consumer; import java.util.stream.Collectors; -import javax.annotation.Nonnull; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.extractCachedUrlIfNeeded; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObjectOrThrow; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isGoogleURL; +import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps; public final class YoutubeMetaInfoHelper { + private static final String ACTION_TEXT = "actionText"; private YoutubeMetaInfoHelper() { } - @Nonnull public static List getMetaInfo(@Nonnull final JsonArray contents) throws ParsingException { @@ -67,17 +64,15 @@ public static List getMetaInfo(@Nonnull final JsonArray contents) private static MetaInfo getInfoPanelContent(@Nonnull final JsonObject infoPanelContentRenderer) throws ParsingException { final MetaInfo metaInfo = new MetaInfo(); - final StringBuilder sb = new StringBuilder(); - for (final Object paragraph : infoPanelContentRenderer.getArray("paragraphs")) { - if (sb.length() != 0) { - sb.append("
"); - } - sb.append(getTextFromObject((JsonObject) paragraph)); - } - metaInfo.setContent(new Description(sb.toString(), Description.HTML)); + final String description = infoPanelContentRenderer.getArray("paragraphs") + .streamAsJsonObjects() + .flatMap(paragraph -> getTextFromObject(paragraph).stream()) + .collect(Collectors.joining("
")); + metaInfo.setContent(new Description(description, Description.HTML)); if (infoPanelContentRenderer.has("sourceEndpoint")) { final String metaInfoLinkUrl = getUrlFromNavigationEndpoint( - infoPanelContentRenderer.getObject("sourceEndpoint")); + infoPanelContentRenderer.getObject("sourceEndpoint")) + .orElse(""); try { metaInfo.addUrl(new URL(Objects.requireNonNull(extractCachedUrlIfNeeded( metaInfoLinkUrl)))); @@ -85,17 +80,10 @@ private static MetaInfo getInfoPanelContent(@Nonnull final JsonObject infoPanelC throw new ParsingException("Could not get metadata info URL", e); } - final String metaInfoLinkText; - if (infoPanelContentRenderer.has("inlineSource")) { - metaInfoLinkText = getTextFromObject( - infoPanelContentRenderer.getObject("inlineSource")); - } else { - metaInfoLinkText = getTextFromObject( - infoPanelContentRenderer.getObject("disclaimer")); - } - if (isNullOrEmpty(metaInfoLinkText)) { - throw new ParsingException("Could not get metadata info link text."); - } + final var source = infoPanelContentRenderer.getObject("inlineSource", + infoPanelContentRenderer.getObject("disclaimer")); + final String metaInfoLinkText = getTextFromObjectOrThrow(source, + "metadata info link"); metaInfo.addUrlText(metaInfoLinkText); } @@ -107,13 +95,10 @@ private static MetaInfo getClarificationRenderer( @Nonnull final JsonObject clarificationRenderer) throws ParsingException { final MetaInfo metaInfo = new MetaInfo(); - final String title = getTextFromObject(clarificationRenderer - .getObject("contentTitle")); - final String text = getTextFromObject(clarificationRenderer - .getObject("text")); - if (title == null || text == null) { - throw new ParsingException("Could not extract clarification renderer content"); - } + final String title = getTextFromObjectOrThrow( + clarificationRenderer.getObject("contentTitle"), "clarification renderer"); + final String text = getTextFromObjectOrThrow( + clarificationRenderer.getObject("text"), "clarification renderer"); metaInfo.setTitle(title); metaInfo.setContent(new Description(text, Description.PLAIN_TEXT)); @@ -122,31 +107,28 @@ private static MetaInfo getClarificationRenderer( .getObject("buttonRenderer"); try { final String url = getUrlFromNavigationEndpoint(actionButton - .getObject("command")); + .getObject("command")).orElse(""); metaInfo.addUrl(new URL(Objects.requireNonNull(extractCachedUrlIfNeeded(url)))); } catch (final NullPointerException | MalformedURLException e) { throw new ParsingException("Could not get metadata info URL", e); } - final String metaInfoLinkText = getTextFromObject( - actionButton.getObject("text")); - if (isNullOrEmpty(metaInfoLinkText)) { - throw new ParsingException("Could not get metadata info link text."); - } - metaInfo.addUrlText(metaInfoLinkText); + final String linkText = getTextFromObjectOrThrow(actionButton.getObject("text"), + "metadata info link"); + metaInfo.addUrlText(linkText); } if (clarificationRenderer.has("secondaryEndpoint") && clarificationRenderer .has("secondarySource")) { final String url = getUrlFromNavigationEndpoint(clarificationRenderer - .getObject("secondaryEndpoint")); + .getObject("secondaryEndpoint")).orElse(null); // Ignore Google URLs, because those point to a Google search about "Covid-19" if (url != null && !isGoogleURL(url)) { try { metaInfo.addUrl(new URL(url)); - final String description = getTextFromObject(clarificationRenderer - .getObject("secondarySource")); - metaInfo.addUrlText(description == null ? url : description); + final String urlText = getTextFromObject(clarificationRenderer + .getObject("secondarySource")).orElse(url); + metaInfo.addUrlText(urlText); } catch (final MalformedURLException e) { throw new ParsingException("Could not get metadata info secondary URL", e); } @@ -179,8 +161,8 @@ private static void getEmergencyOneboxRenderer( // usually a phone number final String action; // this variable is expected to start with "\n" - if (r.has("actionText")) { - action = "\n" + getTextFromObjectOrThrow(r.getObject("actionText"), "action"); + if (r.has(ACTION_TEXT)) { + action = "\n" + getTextFromObjectOrThrow(r.getObject(ACTION_TEXT), "action"); } else if (r.has("contacts")) { final JsonArray contacts = r.getArray("contacts"); final StringBuilder stringBuilder = new StringBuilder(); @@ -188,7 +170,7 @@ private static void getEmergencyOneboxRenderer( for (int i = 0; i < contacts.size(); i++) { stringBuilder.append("\n"); stringBuilder.append(getTextFromObjectOrThrow(contacts.getObject(i) - .getObject("actionText"), "contacts.actionText")); + .getObject(ACTION_TEXT), "contacts.actionText")); } action = stringBuilder.toString(); } else { @@ -207,10 +189,9 @@ private static void getEmergencyOneboxRenderer( metaInfo.addUrlText(urlText); // usually the webpage of the association - final String url = getUrlFromNavigationEndpoint(r.getObject("navigationEndpoint")); - if (url == null) { - throw new ParsingException("Could not extract emergency renderer url"); - } + final String url = getUrlFromNavigationEndpoint(r.getObject("navigationEndpoint")) + .orElseThrow(() -> + new ParsingException("Could not extract emergency renderer url")); try { metaInfo.addUrl(new URL(replaceHttpWithHttps(url))); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java index 30df0b21bf..a11115ec4d 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java @@ -20,30 +20,12 @@ package org.schabi.newpipe.extractor.services.youtube; -import static org.schabi.newpipe.extractor.NewPipe.getDownloader; -import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.ANDROID_CLIENT_VERSION; -import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.DESKTOP_CLIENT_PLATFORM; -import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.IOS_CLIENT_VERSION; -import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.IOS_DEVICE_MODEL; -import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.IOS_USER_AGENT_VERSION; -import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WEB_CLIENT_ID; -import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WEB_CLIENT_NAME; -import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WEB_HARDCODED_CLIENT_VERSION; -import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WEB_REMIX_CLIENT_ID; -import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WEB_REMIX_CLIENT_NAME; -import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WEB_REMIX_HARDCODED_CLIENT_VERSION; -import static org.schabi.newpipe.extractor.utils.Utils.HTTP; -import static org.schabi.newpipe.extractor.utils.Utils.HTTPS; -import static org.schabi.newpipe.extractor.utils.Utils.getStringResultFromRegexArray; -import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; - import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonBuilder; import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; import com.grack.nanojson.JsonWriter; - import org.jsoup.nodes.Entities; import org.schabi.newpipe.extractor.Image; import org.schabi.newpipe.extractor.Image.ResolutionLevel; @@ -62,6 +44,8 @@ import org.schabi.newpipe.extractor.utils.RandomStringFromAlphabetGenerator; import org.schabi.newpipe.extractor.utils.Utils; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; import java.io.IOException; import java.net.MalformedURLException; import java.net.URL; @@ -73,14 +57,32 @@ import java.util.Optional; import java.util.Random; import java.util.Set; +import java.util.function.Predicate; import java.util.regex.Pattern; import java.util.stream.Collectors; import java.util.stream.Stream; -import javax.annotation.Nonnull; -import javax.annotation.Nullable; +import static org.schabi.newpipe.extractor.NewPipe.getDownloader; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.ANDROID_CLIENT_VERSION; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.DESKTOP_CLIENT_PLATFORM; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.IOS_CLIENT_VERSION; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.IOS_DEVICE_MODEL; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.IOS_USER_AGENT_VERSION; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WEB_CLIENT_ID; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WEB_CLIENT_NAME; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WEB_HARDCODED_CLIENT_VERSION; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WEB_REMIX_CLIENT_ID; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WEB_REMIX_CLIENT_NAME; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WEB_REMIX_HARDCODED_CLIENT_VERSION; +import static org.schabi.newpipe.extractor.utils.Utils.HTTP; +import static org.schabi.newpipe.extractor.utils.Utils.HTTPS; +import static org.schabi.newpipe.extractor.utils.Utils.getStringResultFromRegexArray; +import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; public final class YoutubeParsingHelper { + public static final String NAVIGATION_ENDPOINT = "navigationEndpoint"; + public static final String SUBSCRIBER_COUNT_TEXT = "subscriberCountText"; + public static final String TITLE = "title"; private YoutubeParsingHelper() { } @@ -194,6 +196,8 @@ private YoutubeParsingHelper() { private static boolean consentAccepted = false; + public static final Predicate STRING_PREDICATE = text -> !text.isBlank(); + public static boolean isGoogleURL(final String url) { final String cachedUrl = extractCachedUrlIfNeeded(url); try { @@ -550,8 +554,8 @@ private static String getClientVersionFromServiceTrackingParam( .streamAsJsonObjects()) .filter(param -> param.getString("key", "") .equals(clientVersionKey)) - .map(param -> param.getString("value")) - .filter(paramValue -> !isNullOrEmpty(paramValue)) + .map(param -> param.getString("value", "")) + .filter(STRING_PREDICATE) .findFirst() .orElse(null); } @@ -684,103 +688,110 @@ public static String getYoutubeMusicClientVersion() return youtubeMusicClientVersion; } - @Nullable - public static String getUrlFromNavigationEndpoint( + @Nonnull + public static Optional getUrlFromNavigationEndpoint( @Nonnull final JsonObject navigationEndpoint) { - if (navigationEndpoint.has("urlEndpoint")) { - String internUrl = navigationEndpoint.getObject("urlEndpoint") - .getString("url"); - if (internUrl.startsWith("https://www.youtube.com/redirect?")) { - // remove https://www.youtube.com part to fall in the next if block - internUrl = internUrl.substring(23); - } + return YoutubeParsingHelper.extractUrlFromUrlEndpoint(navigationEndpoint) + .or(() -> Optional.ofNullable(navigationEndpoint.getObject("browseEndpoint", null)) + .map(browseEndpoint -> { + final var canonicalBaseUrl = + browseEndpoint.getString("canonicalBaseUrl"); + final var browseId = browseEndpoint.getString("browseId"); + + if (browseId != null) { + if (browseId.startsWith("UC")) { + // All channel IDs are prefixed with UC + return "https://www.youtube.com/channel/" + browseId; + } else if (browseId.startsWith("VL")) { + // All playlist IDs are prefixed with VL, which needs to be + // removed from the playlist ID + return "https://www.youtube.com/playlist?list=" + + browseId.substring(2); + } + } else if (!isNullOrEmpty(canonicalBaseUrl)) { + return "https://www.youtube.com" + canonicalBaseUrl; + } + return null; + })) + .or(() -> Optional.ofNullable(navigationEndpoint.getObject("watchEndpoint", null)) + .map(watchEndpoint -> { + final var videoId = watchEndpoint.getString(VIDEO_ID); + final var playlistId = watchEndpoint.getString("playlistId"); + final var startTime = watchEndpoint.getInt("startTimeSeconds", -1); + return "https://www.youtube.com/watch?v=" + videoId + + (playlistId != null ? "&list=" + playlistId : "") + + (startTime != -1 ? "&t=" + startTime : ""); + })) + .or(() -> { + final var playlistId = navigationEndpoint.getObject("watchPlaylistEndpoint") + .getString("playlistId"); + return Optional.ofNullable(playlistId) + .map(id -> "https://www.youtube.com/playlist?list=" + id); + }) + .or(() -> { + final var listItems = navigationEndpoint.getObject("showDialogCommand") + .getObject("panelLoadingStrategy").getObject("inlineContent") + .getObject("dialogViewModel").getObject("customContent") + .getObject("listViewModel") + .getArray("listItems"); + + // the first item seems to always be the channel that actually uploaded the + // video, i.e. it appears in their video feed + final var command = listItems.getObject(0).getObject("listItemViewModel") + .getObject("rendererContext").getObject("commandContext") + .getObject("onTap").getObject("innertubeCommand", null); + return Optional.ofNullable(command) + .flatMap(YoutubeParsingHelper::getUrlFromNavigationEndpoint); + }) + .filter(STRING_PREDICATE); + } - if (internUrl.startsWith("/redirect?")) { - // q parameter can be the first parameter - internUrl = internUrl.substring(10); - final String[] params = internUrl.split("&"); - for (final String param : params) { - if (param.split("=")[0].equals("q")) { - return Utils.decodeUrlUtf8(param.split("=")[1]); + @Nonnull + private static Optional extractUrlFromUrlEndpoint(@Nonnull final JsonObject endpoint) { + return Optional.ofNullable(endpoint.getObject("urlEndpoint").getString("url")) + .map(internUrl -> { + if (internUrl.startsWith("https://www.youtube.com/redirect?")) { + // remove https://www.youtube.com part to fall in the next if block + internUrl = internUrl.substring(23); } - } - } else if (internUrl.startsWith("http")) { - return internUrl; - } else if (internUrl.startsWith("/channel") || internUrl.startsWith("/user") - || internUrl.startsWith("/watch")) { - return "https://www.youtube.com" + internUrl; - } - } - - if (navigationEndpoint.has("browseEndpoint")) { - final JsonObject browseEndpoint = navigationEndpoint.getObject("browseEndpoint"); - final String canonicalBaseUrl = browseEndpoint.getString("canonicalBaseUrl"); - final String browseId = browseEndpoint.getString("browseId"); - - if (browseId != null) { - if (browseId.startsWith("UC")) { - // All channel IDs are prefixed with UC - return "https://www.youtube.com/channel/" + browseId; - } else if (browseId.startsWith("VL")) { - // All playlist IDs are prefixed with VL, which needs to be removed from the - // playlist ID - return "https://www.youtube.com/playlist?list=" + browseId.substring(2); - } - } - - if (!isNullOrEmpty(canonicalBaseUrl)) { - return "https://www.youtube.com" + canonicalBaseUrl; - } - } - - if (navigationEndpoint.has("watchEndpoint")) { - final StringBuilder url = new StringBuilder(); - url.append("https://www.youtube.com/watch?v=") - .append(navigationEndpoint.getObject("watchEndpoint") - .getString(VIDEO_ID)); - if (navigationEndpoint.getObject("watchEndpoint").has("playlistId")) { - url.append("&list=").append(navigationEndpoint.getObject("watchEndpoint") - .getString("playlistId")); - } - if (navigationEndpoint.getObject("watchEndpoint").has("startTimeSeconds")) { - url.append("&t=") - .append(navigationEndpoint.getObject("watchEndpoint") - .getInt("startTimeSeconds")); - } - return url.toString(); - } - - if (navigationEndpoint.has("watchPlaylistEndpoint")) { - return "https://www.youtube.com/playlist?list=" - + navigationEndpoint.getObject("watchPlaylistEndpoint") - .getString("playlistId"); - } - - if (navigationEndpoint.has("showDialogCommand")) { - try { - final JsonArray listItems = JsonUtils.getArray(navigationEndpoint, - "showDialogCommand.panelLoadingStrategy.inlineContent.dialogViewModel" - + ".customContent.listViewModel.listItems"); - - // the first item seems to always be the channel that actually uploaded the video, - // i.e. it appears in their video feed - final JsonObject command = JsonUtils.getObject(listItems.getObject(0), - "listItemViewModel.rendererContext.commandContext.onTap.innertubeCommand"); - return getUrlFromNavigationEndpoint(command); - } catch (final ParsingException p) { - } - } + if (internUrl.startsWith("/redirect?")) { + // q parameter can be the first parameter + internUrl = internUrl.substring(10); + final String[] params = internUrl.split("&"); + for (final String param : params) { + final String[] nameAndValue = param.split("="); + if (nameAndValue[0].equals("q")) { + return Utils.decodeUrlUtf8(nameAndValue[1]); + } + } + } else if (internUrl.startsWith("http")) { + return internUrl; + } else if (internUrl.startsWith("/channel") || internUrl.startsWith("/user") + || internUrl.startsWith("/watch")) { + return "https://www.youtube.com" + internUrl; + } - if (navigationEndpoint.has("commandMetadata")) { - final JsonObject metadata = navigationEndpoint.getObject("commandMetadata") - .getObject("webCommandMetadata"); - if (metadata.has("url")) { - return "https://www.youtube.com" + metadata.getString("url"); - } - } + return null; + }); + } - return null; + @Nonnull + public static Optional getMusicUploaderUrlFromMenu(@Nonnull final JsonObject object) { + final var items = object.getObject("menu") + .getObject("menuRenderer") + .getArray("items"); + return items.streamAsJsonObjects() + .flatMap(item -> { + final var renderer = item.getObject("menuNavigationItemRenderer"); + final var iconType = renderer.getObject("icon").getString("iconType"); + final var endpoint = renderer.getObject(NAVIGATION_ENDPOINT); + + return "ARTIST".equals(iconType) + ? getUrlFromNavigationEndpoint(endpoint).stream() + : Stream.empty(); + }) + .findFirst(); } /** @@ -789,76 +800,48 @@ public static String getUrlFromNavigationEndpoint( * * @param textObject JSON object to get the text from * @param html whether to return HTML, by parsing the {@code navigationEndpoint} - * @return text in the JSON object or {@code null} + * @return text in the JSON object as an {@link Optional} */ - @Nullable - public static String getTextFromObject(final JsonObject textObject, final boolean html) { - if (isNullOrEmpty(textObject)) { - return null; - } - - if (textObject.has("simpleText")) { - return textObject.getString("simpleText"); - } - - final JsonArray runs = textObject.getArray("runs"); - if (runs.isEmpty()) { - return null; - } - - final StringBuilder textBuilder = new StringBuilder(); - for (final Object o : runs) { - final JsonObject run = (JsonObject) o; - String text = run.getString("text"); - - if (html) { - if (run.has("navigationEndpoint")) { - final String url = getUrlFromNavigationEndpoint( - run.getObject("navigationEndpoint")); - if (!isNullOrEmpty(url)) { - text = "" + Entities.escape(text) - + ""; + @Nonnull + public static Optional getTextFromObject(@Nonnull final JsonObject textObject, + final boolean html) { + return Optional.ofNullable(textObject.getString("simpleText")) + .or(() -> Optional.of(extractRuns(textObject, html))) + .filter(STRING_PREDICATE); + } + + private static String extractRuns(@Nonnull final JsonObject textObject, final boolean html) { + final var runs = textObject.getArray("runs"); + final String text = runs.streamAsJsonObjects() + .map(run -> { + String textString = run.getString("text"); + + if (html) { + final String url = getUrlFromNavigationEndpoint( + run.getObject(NAVIGATION_ENDPOINT)).orElse(null); + if (url != null) { + textString = "" + + Entities.escape(textString) + ""; + } + + if (run.getBoolean("strikethrough")) { + textString = "" + textString + ""; + } + if (run.getBoolean("italics")) { + textString = "" + textString + ""; + } + if (run.getBoolean("bold")) { + textString = "" + textString + ""; + } } - } - - final boolean bold = run.has("bold") - && run.getBoolean("bold"); - final boolean italic = run.has("italics") - && run.getBoolean("italics"); - final boolean strikethrough = run.has("strikethrough") - && run.getBoolean("strikethrough"); - - if (bold) { - textBuilder.append(""); - } - if (italic) { - textBuilder.append(""); - } - if (strikethrough) { - textBuilder.append(""); - } - - textBuilder.append(text); - - if (strikethrough) { - textBuilder.append(""); - } - if (italic) { - textBuilder.append(""); - } - if (bold) { - textBuilder.append(""); - } - } else { - textBuilder.append(text); - } - } - String text = textBuilder.toString(); + return textString; + }) + .collect(Collectors.joining()); if (html) { - text = text.replaceAll("\\n", "
"); - text = text.replaceAll(" {2}", "  "); + return text.replace("\\n", "
") + .replaceAll(" {2}", "  "); } return text; @@ -867,47 +850,30 @@ public static String getTextFromObject(final JsonObject textObject, final boolea @Nonnull public static String getTextFromObjectOrThrow(final JsonObject textObject, final String error) throws ParsingException { - final String result = getTextFromObject(textObject); - if (result == null) { - throw new ParsingException("Could not extract text: " + error); - } - return result; + return getTextFromObject(textObject) + .orElseThrow(() -> new ParsingException("Could not extract text: " + error)); } - @Nullable - public static String getTextFromObject(final JsonObject textObject) { + @Nonnull + public static Optional getTextFromObject(@Nonnull final JsonObject textObject) { return getTextFromObject(textObject, false); } - @Nullable - public static String getUrlFromObject(final JsonObject textObject) { - if (isNullOrEmpty(textObject)) { - return null; - } - - final JsonArray runs = textObject.getArray("runs"); - if (runs.isEmpty()) { - return null; - } - - for (final Object textPart : runs) { - final String url = getUrlFromNavigationEndpoint(((JsonObject) textPart) - .getObject("navigationEndpoint")); - if (!isNullOrEmpty(url)) { - return url; - } - } - - return null; + @Nonnull + public static Optional getUrlFromObject(@Nonnull final JsonObject textObject) { + return textObject.getArray("runs").streamAsJsonObjects() + .flatMap(textPart -> getUrlFromNavigationEndpoint(textPart + .getObject(NAVIGATION_ENDPOINT)) + .stream()) + .findFirst(); } - @Nullable - public static String getTextAtKey(@Nonnull final JsonObject jsonObject, final String theKey) { - if (jsonObject.isString(theKey)) { - return jsonObject.getString(theKey); - } else { - return getTextFromObject(jsonObject.getObject(theKey)); - } + @Nonnull + public static Optional getTextAtKey(@Nonnull final JsonObject jsonObject, + final String theKey) { + return Optional.ofNullable(jsonObject.getString(theKey)) + .filter(STRING_PREDICATE) + .or(() -> getTextFromObject(jsonObject.getObject(theKey))); } public static String fixThumbnailUrl(@Nonnull final String thumbnailUrl) { @@ -1222,7 +1188,8 @@ public static void defaultAlertsCheck(@Nonnull final JsonObject initialData) final JsonArray alerts = initialData.getArray("alerts"); if (!isNullOrEmpty(alerts)) { final JsonObject alertRenderer = alerts.getObject(0).getObject("alertRenderer"); - final String alertText = getTextFromObject(alertRenderer.getObject("text")); + final String alertText = getTextFromObject(alertRenderer.getObject("text")) + .orElse(null); final String alertType = alertRenderer.getString("type", ""); if (alertType.equalsIgnoreCase("ERROR")) { if (alertText != null @@ -1279,21 +1246,14 @@ public static String extractCachedUrlIfNeeded(final String url) { return url; } - public static boolean isVerified(final JsonArray badges) { - if (Utils.isNullOrEmpty(badges)) { - return false; - } - - for (final Object badge : badges) { - final String style = ((JsonObject) badge).getObject("metadataBadgeRenderer") - .getString("style"); - if (style != null && (style.equals("BADGE_STYLE_TYPE_VERIFIED") - || style.equals("BADGE_STYLE_TYPE_VERIFIED_ARTIST"))) { - return true; - } - } - - return false; + public static boolean isVerified(@Nonnull final JsonArray badges) { + return badges.streamAsJsonObjects() + .anyMatch(badge -> { + final String style = badge.getObject("metadataBadgeRenderer") + .getString("style"); + return "BADGE_STYLE_TYPE_VERIFIED".equals(style) + || "BADGE_STYLE_TYPE_VERIFIED_ARTIST".equals(style); + }); } public static boolean hasArtistOrVerifiedIconBadgeAttachment( @@ -1562,19 +1522,18 @@ public static JsonBuilder prepareJsonBuilder( * Gets the first collaborator, which is the channel that owns the video, * i.e. the video is displayed on their channel page. * - * @param navigationEndpoint JSON object for the navigationEndpoint - * @return The first collaborator in the JSON object or {@code null} + * @param renderer JSON object for the video renderer + * @return An {@link Optional} containing the first collaborator, if one is present */ - @Nullable - public static JsonObject getFirstCollaborator(final JsonObject navigationEndpoint) - throws ParsingException { + @Nonnull + public static Optional getFirstCollaborator(final JsonObject renderer) { try { // CHECKSTYLE:OFF - final JsonArray listItems = JsonUtils.getArray(navigationEndpoint, "showDialogCommand.panelLoadingStrategy.inlineContent.dialogViewModel.customContent.listViewModel.listItems"); + final JsonArray listItems = JsonUtils.getArray(renderer, "showDialogCommand.panelLoadingStrategy.inlineContent.dialogViewModel.customContent.listViewModel.listItems"); // CHECKSTYLE:ON - return listItems.getObject(0).getObject("listItemViewModel"); + return Optional.ofNullable(listItems.getObject(0).getObject("listItemViewModel", null)); } catch (final ParsingException e) { - return null; + return Optional.empty(); } } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeBaseShowInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeBaseShowInfoItemExtractor.java index 67254302fc..8499abe12e 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeBaseShowInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeBaseShowInfoItemExtractor.java @@ -9,7 +9,7 @@ import javax.annotation.Nonnull; import java.util.List; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObjectOrThrow; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getThumbnailsFromInfoItem; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint; @@ -32,7 +32,8 @@ public String getName() throws ParsingException { @Override public String getUrl() throws ParsingException { - return getUrlFromNavigationEndpoint(showRenderer.getObject("navigationEndpoint")); + return getUrlFromNavigationEndpoint(showRenderer.getObject("navigationEndpoint")) + .orElse(null); } @Nonnull @@ -46,13 +47,10 @@ public List getThumbnails() throws ParsingException { public long getStreamCount() throws ParsingException { // The stream count should be always returned in the first text object for English // localizations, but the complete text is parsed for reliability purposes - final String streamCountText = getTextFromObject( - showRenderer.getObject("thumbnailOverlays") - .getObject("thumbnailOverlayBottomPanelRenderer") - .getObject("text")); - if (streamCountText == null) { - throw new ParsingException("Could not get stream count"); - } + final var textObject = showRenderer.getObject("thumbnailOverlays") + .getObject("thumbnailOverlayBottomPanelRenderer") + .getObject("text"); + final String streamCountText = getTextFromObjectOrThrow(textObject, "stream count"); try { // The data returned could be a human/shortened number, but no show with more than 1000 diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java index afbba366f5..3962be8e9f 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java @@ -257,7 +257,7 @@ public long getSubscriberCount() throws ParsingException { if (textObject != null) { try { - return Utils.mixedNumberWordToLong(getTextFromObject(textObject)); + return Utils.mixedNumberWordToLong(getTextFromObject(textObject).orElse("")); } catch (final NumberFormatException e) { throw new ParsingException("Could not get subscriber count", e); } @@ -321,7 +321,8 @@ public String getDescription() throws ParsingException { The description extracted is incomplete and the original one can be only accessed from the About tab */ - return getTextFromObject(channelHeader.json.getObject("description")); + return getTextFromObject(channelHeader.json.getObject("description")) + .orElse(null); } return jsonResponse.getObject(METADATA) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java index 38861f4e51..d02919c6b8 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java @@ -34,9 +34,13 @@ import java.util.List; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObjectOrThrow; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getThumbnailsFromInfoItem; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.SUBSCRIBER_COUNT_TEXT; public class YoutubeChannelInfoItemExtractor implements ChannelInfoItemExtractor { + private static final String VIDEO_COUNT_TEXT = "videoCountText"; + private final JsonObject channelInfoItem; /** * New layout: @@ -47,14 +51,9 @@ public class YoutubeChannelInfoItemExtractor implements ChannelInfoItemExtractor public YoutubeChannelInfoItemExtractor(final JsonObject channelInfoItem) { this.channelInfoItem = channelInfoItem; - - boolean wHandle = false; - final String subscriberCountText = getTextFromObject( - channelInfoItem.getObject("subscriberCountText")); - if (subscriberCountText != null) { - wHandle = subscriberCountText.startsWith("@"); - } - this.withHandle = wHandle; + this.withHandle = getTextFromObject(channelInfoItem.getObject(SUBSCRIBER_COUNT_TEXT)) + .map(text -> text.startsWith("@")) + .orElse(false); } @Nonnull @@ -69,11 +68,7 @@ public List getThumbnails() throws ParsingException { @Override public String getName() throws ParsingException { - try { - return getTextFromObject(channelInfoItem.getObject("title")); - } catch (final Exception e) { - throw new ParsingException("Could not get name", e); - } + return getTextFromObjectOrThrow(channelInfoItem.getObject("title"), "name"); } @Override @@ -89,22 +84,22 @@ public String getUrl() throws ParsingException { @Override public long getSubscriberCount() throws ParsingException { try { - if (!channelInfoItem.has("subscriberCountText")) { + if (!channelInfoItem.has(SUBSCRIBER_COUNT_TEXT)) { // Subscription count is not available for this channel item. return -1; } if (withHandle) { - if (channelInfoItem.has("videoCountText")) { + if (channelInfoItem.has(VIDEO_COUNT_TEXT)) { return Utils.mixedNumberWordToLong(getTextFromObject( - channelInfoItem.getObject("videoCountText"))); + channelInfoItem.getObject(VIDEO_COUNT_TEXT)).orElse("")); } else { return -1; } } return Utils.mixedNumberWordToLong(getTextFromObject( - channelInfoItem.getObject("subscriberCountText"))); + channelInfoItem.getObject(SUBSCRIBER_COUNT_TEXT)).orElse("")); } catch (final Exception e) { throw new ParsingException("Could not get subscriber count", e); } @@ -113,14 +108,14 @@ public long getSubscriberCount() throws ParsingException { @Override public long getStreamCount() throws ParsingException { try { - if (withHandle || !channelInfoItem.has("videoCountText")) { + if (withHandle || !channelInfoItem.has(VIDEO_COUNT_TEXT)) { // Video count is not available, either the channel has no public uploads // or YouTube displays the channel handle instead. return ListExtractor.ITEM_COUNT_UNKNOWN; } return Long.parseLong(Utils.removeNonDigitCharacters(getTextFromObject( - channelInfoItem.getObject("videoCountText")))); + channelInfoItem.getObject(VIDEO_COUNT_TEXT)).orElse(""))); } catch (final Exception e) { throw new ParsingException("Could not get stream count", e); } @@ -133,15 +128,12 @@ public boolean isVerified() throws ParsingException { @Override public String getDescription() throws ParsingException { - try { - if (!channelInfoItem.has("descriptionSnippet")) { - // Channel have no description. - return null; - } - - return getTextFromObject(channelInfoItem.getObject("descriptionSnippet")); - } catch (final Exception e) { - throw new ParsingException("Could not get description", e); + if (!channelInfoItem.has("descriptionSnippet")) { + // Channel have no description. + return null; } + + return getTextFromObjectOrThrow(channelInfoItem.getObject("descriptionSnippet"), + "description"); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java index a4e5d498ee..f5fe68833a 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java @@ -388,7 +388,7 @@ public int getCommentsCount() throws ExtractionException { try { return Integer.parseInt( - Utils.removeNonDigitCharacters(getTextFromObject(countText)) + Utils.removeNonDigitCharacters(getTextFromObject(countText).orElse("")) ); } catch (final Exception e) { throw new ExtractionException("Unable to get comments count", e); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsInfoItemExtractor.java index ddc7b7bcc0..45371f9035 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsInfoItemExtractor.java @@ -66,7 +66,8 @@ public List getThumbnails() throws ParsingException { @Override public String getName() throws ParsingException { try { - return getTextFromObject(JsonUtils.getObject(commentRenderer, "authorText")); + return getTextFromObject(JsonUtils.getObject(commentRenderer, "authorText")) + .orElse(""); } catch (final Exception e) { return ""; } @@ -75,8 +76,8 @@ public String getName() throws ParsingException { @Override public String getTextualUploadDate() throws ParsingException { try { - return getTextFromObject(JsonUtils.getObject(commentRenderer, - "publishedTimeText")); + return getTextFromObject(JsonUtils.getObject(commentRenderer, "publishedTimeText")) + .orElse(null); } catch (final Exception e) { throw new ParsingException("Could not get publishedTimeText", e); } @@ -170,10 +171,7 @@ public String getTextualLikeCount() throws ParsingException { } final JsonObject voteCountObj = JsonUtils.getObject(commentRenderer, "voteCount"); - if (voteCountObj.isEmpty()) { - return ""; - } - return getTextFromObject(voteCountObj); + return getTextFromObject(voteCountObj).orElse(""); } catch (final Exception e) { throw new ParsingException("Could not get the vote count", e); } @@ -189,7 +187,7 @@ public Description getCommentText() throws ParsingException { // https://github.com/TeamNewPipe/NewPipeExtractor/issues/380#issuecomment-668808584 return Description.EMPTY_DESCRIPTION; } - final String commentText = getTextFromObject(contentText, true); + final String commentText = getTextFromObject(contentText, true).orElse(""); // YouTube adds U+FEFF in some comments. // eg. https://www.youtube.com/watch?v=Nj4F63E59io final String commentTextBomRemoved = Utils.removeUTF8BOM(commentText); @@ -235,7 +233,8 @@ public boolean isUploaderVerified() throws ParsingException { @Override public String getUploaderName() throws ParsingException { try { - return getTextFromObject(JsonUtils.getObject(commentRenderer, "authorText")); + return getTextFromObject(JsonUtils.getObject(commentRenderer, "authorText")) + .orElse(""); } catch (final Exception e) { return ""; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixOrPlaylistInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixOrPlaylistInfoItemExtractor.java index 16c7a3e3ea..48cc1fab97 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixOrPlaylistInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixOrPlaylistInfoItemExtractor.java @@ -1,7 +1,7 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.extractPlaylistTypeFromPlaylistUrl; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObjectOrThrow; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getThumbnailsFromInfoItem; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; @@ -26,11 +26,7 @@ public YoutubeMixOrPlaylistInfoItemExtractor(final JsonObject mixInfoItem) { @Override public String getName() throws ParsingException { - final String name = getTextFromObject(mixInfoItem.getObject("title")); - if (isNullOrEmpty(name)) { - throw new ParsingException("Could not get name"); - } - return name; + return getTextFromObjectOrThrow(mixInfoItem.getObject("title"), "name"); } @Override @@ -51,7 +47,8 @@ public List getThumbnails() throws ParsingException { @Override public String getUploaderName() throws ParsingException { // this will be a list of uploaders for mixes - return YoutubeParsingHelper.getTextFromObject(mixInfoItem.getObject("longBylineText")); + return YoutubeParsingHelper.getTextFromObject(mixInfoItem.getObject("longBylineText")) + .orElse(null); } @Override @@ -68,11 +65,9 @@ public boolean isUploaderVerified() throws ParsingException { @Override public long getStreamCount() throws ParsingException { - final String countString = YoutubeParsingHelper.getTextFromObject( - mixInfoItem.getObject("videoCountShortText")); - if (countString == null) { - throw new ParsingException("Could not extract item count for playlist/mix info item"); - } + final var textObject = mixInfoItem.getObject("videoCountShortText"); + final String countString = getTextFromObjectOrThrow(textObject, + "item count for playlist/mix info item"); try { return Integer.parseInt(countString); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixOrPlaylistLockupInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixOrPlaylistLockupInfoItemExtractor.java index 9b90552503..a45d4c393b 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixOrPlaylistLockupInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixOrPlaylistLockupInfoItemExtractor.java @@ -87,7 +87,8 @@ public String getUploaderUrl() throws ParsingException { .getArray("commandRuns") .getObject(0) .getObject("onTap") - .getObject("innertubeCommand")); + .getObject("innertubeCommand")) + .orElse(null); } @Override @@ -155,7 +156,7 @@ public String getUrl() throws ParsingException { return getUrlFromNavigationEndpoint(lockupViewModel.getObject("rendererContext") .getObject("commandContext") .getObject("onTap") - .getObject("innertubeCommand")); + .getObject("innertubeCommand")).orElse(null); } @Nonnull diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java index bfc21d9ccf..3648b85e12 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java @@ -125,11 +125,8 @@ public void onFetchPage(@Nonnull final Downloader downloader) @Nonnull @Override public String getName() throws ParsingException { - final String name = YoutubeParsingHelper.getTextAtKey(playlistData, "title"); - if (isNullOrEmpty(name)) { - throw new ParsingException("Could not get playlist name"); - } - return name; + return YoutubeParsingHelper.getTextAtKey(playlistData, "title") + .orElseThrow(() -> new ParsingException("Could not get playlist name")); } @Nonnull diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicAlbumOrPlaylistInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicAlbumOrPlaylistInfoItemExtractor.java index 1b3fccfb27..138043f428 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicAlbumOrPlaylistInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicAlbumOrPlaylistInfoItemExtractor.java @@ -11,8 +11,9 @@ import java.util.List; import static org.schabi.newpipe.extractor.ListExtractor.ITEM_COUNT_UNKNOWN; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getImagesFromThumbnailsArray; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getMusicUploaderUrlFromMenu; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObjectOrThrow; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_ALBUMS; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; @@ -49,16 +50,11 @@ public List getThumbnails() throws ParsingException { @Override public String getName() throws ParsingException { - final String name = getTextFromObject(albumOrPlaylistInfoItem.getArray("flexColumns") + final var textObject = albumOrPlaylistInfoItem.getArray("flexColumns") .getObject(0) .getObject("musicResponsiveListItemFlexColumnRenderer") - .getObject("text")); - - if (!isNullOrEmpty(name)) { - return name; - } - - throw new ParsingException("Could not get name"); + .getObject("text"); + return getTextFromObjectOrThrow(textObject, "name"); } @Override @@ -105,28 +101,13 @@ public String getUploaderName() throws ParsingException { @Override public String getUploaderUrl() throws ParsingException { // first try obtaining the uploader from the menu (will not work for MUSIC_PLAYLISTS though) - final JsonArray items = albumOrPlaylistInfoItem.getObject("menu") - .getObject("menuRenderer") - .getArray("items"); - for (final Object item : items) { - final JsonObject menuNavigationItemRenderer = - ((JsonObject) item).getObject("menuNavigationItemRenderer"); - if (menuNavigationItemRenderer.getObject("icon") - .getString("iconType", "") - .equals("ARTIST")) { - return getUrlFromNavigationEndpoint( - menuNavigationItemRenderer.getObject("navigationEndpoint")); - } - } - - // then try obtaining it from the uploader description element - if (!descriptionElementUploader.has("navigationEndpoint")) { - // if there is no navigationEndpoint for the uploader - // then this playlist/album is likely autogenerated - return null; - } - return getUrlFromNavigationEndpoint( - descriptionElementUploader.getObject("navigationEndpoint")); + return getMusicUploaderUrlFromMenu(albumOrPlaylistInfoItem) + // then try obtaining it from the uploader description element + .or(() -> getUrlFromNavigationEndpoint(descriptionElementUploader + .getObject("navigationEndpoint"))) + // if there is no navigationEndpoint for the uploader + // then this playlist/album is likely autogenerated + .orElse(null); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicArtistInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicArtistInfoItemExtractor.java index 4fe2dc666f..4251fbc5bd 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicArtistInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicArtistInfoItemExtractor.java @@ -12,7 +12,7 @@ import javax.annotation.Nullable; import java.util.List; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObjectOrThrow; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getImagesFromThumbnailsArray; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; @@ -40,24 +40,17 @@ public List getThumbnails() throws ParsingException { @Override public String getName() throws ParsingException { - final String name = getTextFromObject(artistInfoItem.getArray("flexColumns") + final var jsonObject = artistInfoItem.getArray("flexColumns") .getObject(0) .getObject("musicResponsiveListItemFlexColumnRenderer") - .getObject("text")); - if (!isNullOrEmpty(name)) { - return name; - } - throw new ParsingException("Could not get name"); + .getObject("text"); + return getTextFromObjectOrThrow(jsonObject, "name"); } @Override public String getUrl() throws ParsingException { - final String url = getUrlFromNavigationEndpoint( - artistInfoItem.getObject("navigationEndpoint")); - if (!isNullOrEmpty(url)) { - return url; - } - throw new ParsingException("Could not get URL"); + return getUrlFromNavigationEndpoint(artistInfoItem.getObject("navigationEndpoint")) + .orElseThrow(() -> new ParsingException("Could not get URL")); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java index 8e365b2ff2..b84959766b 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java @@ -149,7 +149,8 @@ public String getSearchSuggestion() throws ParsingException { .getObject("didYouMeanRenderer"); if (!didYouMeanRenderer.isEmpty()) { - return getTextFromObject(didYouMeanRenderer.getObject("correctedQuery")); + return getTextFromObject(didYouMeanRenderer.getObject("correctedQuery")) + .orElse(""); } // NOTE: As of 2025-07 "showing results for ..." doesn't seem to be returned by diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSongOrVideoInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSongOrVideoInfoItemExtractor.java index 11b220288c..35917776c4 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSongOrVideoInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSongOrVideoInfoItemExtractor.java @@ -14,8 +14,9 @@ import javax.annotation.Nonnull; import java.util.List; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getImagesFromThumbnailsArray; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getMusicUploaderUrlFromMenu; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObjectOrThrow; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_SONGS; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_VIDEOS; @@ -46,14 +47,11 @@ public String getUrl() throws ParsingException { @Override public String getName() throws ParsingException { - final String name = getTextFromObject(songOrVideoInfoItem.getArray("flexColumns") + final var textObject = songOrVideoInfoItem.getArray("flexColumns") .getObject(0) .getObject("musicResponsiveListItemFlexColumnRenderer") - .getObject("text")); - if (!isNullOrEmpty(name)) { - return name; - } - throw new ParsingException("Could not get name"); + .getObject("text"); + return getTextFromObjectOrThrow(textObject, "name"); } @Override @@ -88,41 +86,22 @@ public String getUploaderName() throws ParsingException { @Override public String getUploaderUrl() throws ParsingException { if (searchType.equals(MUSIC_VIDEOS)) { - final JsonArray items = songOrVideoInfoItem.getObject("menu") - .getObject("menuRenderer") - .getArray("items"); - for (final Object item : items) { - final JsonObject menuNavigationItemRenderer = - ((JsonObject) item).getObject("menuNavigationItemRenderer"); - if (menuNavigationItemRenderer.getObject("icon") - .getString("iconType", "") - .equals("ARTIST")) { - return getUrlFromNavigationEndpoint( - menuNavigationItemRenderer.getObject("navigationEndpoint")); - } - } - - return null; + return getMusicUploaderUrlFromMenu(songOrVideoInfoItem).orElse(null); } else { - final JsonObject navigationEndpointHolder = songOrVideoInfoItem.getArray("flexColumns") + final var endpoint = songOrVideoInfoItem.getArray("flexColumns") .getObject(1) .getObject("musicResponsiveListItemFlexColumnRenderer") .getObject("text") .getArray("runs") - .getObject(0); + .getObject(0) + .getObject("navigationEndpoint"); - if (!navigationEndpointHolder.has("navigationEndpoint")) { + if (!endpoint.isEmpty()) { + return getUrlFromNavigationEndpoint(endpoint) + .orElseThrow(() -> new ParsingException("Could not get uploader URL")); + } else { return null; } - - final String url = getUrlFromNavigationEndpoint( - navigationEndpointHolder.getObject("navigationEndpoint")); - - if (!isNullOrEmpty(url)) { - return url; - } - - throw new ParsingException("Could not get uploader URL"); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java index 630e8b91ad..3616c905da 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java @@ -1,21 +1,8 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.DISABLE_PRETTY_PRINT_PARAMETER; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.YOUTUBEI_V1_URL; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.extractPlaylistTypeFromPlaylistUrl; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getImagesFromThumbnailsArray; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder; -import static org.schabi.newpipe.extractor.services.youtube.protos.playlist.PlaylistProtobufContinuation.ContinuationParams; -import static org.schabi.newpipe.extractor.services.youtube.protos.playlist.PlaylistProtobufContinuation.PlaylistContinuation; -import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; - import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonWriter; - import org.schabi.newpipe.extractor.Image; import org.schabi.newpipe.extractor.Page; import org.schabi.newpipe.extractor.StreamingService; @@ -33,13 +20,27 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; import org.schabi.newpipe.extractor.utils.Utils; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; import java.io.IOException; import java.nio.charset.StandardCharsets; import java.util.Base64; import java.util.List; +import java.util.Optional; -import javax.annotation.Nonnull; -import javax.annotation.Nullable; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.DISABLE_PRETTY_PRINT_PARAMETER; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.YOUTUBEI_V1_URL; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.extractPlaylistTypeFromPlaylistUrl; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getImagesFromThumbnailsArray; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObjectOrThrow; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder; +import static org.schabi.newpipe.extractor.services.youtube.protos.playlist.PlaylistProtobufContinuation.ContinuationParams; +import static org.schabi.newpipe.extractor.services.youtube.protos.playlist.PlaylistProtobufContinuation.PlaylistContinuation; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.TITLE; +import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; public class YoutubePlaylistExtractor extends PlaylistExtractor { // Names of some objects in JSON response frequently used in this class @@ -52,6 +53,8 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { private static final String MICROFORMAT = "microformat"; // Continuation properties requesting first page and showing unavailable videos private static final String PLAYLIST_CONTINUATION_PROPERTIES_BASE64 = "CADCBgIIAA%3D%3D"; + private static final String THUMBNAIL = "thumbnail"; + private static final String THUMBNAILS = "thumbnails"; private JsonObject browseMetadataResponse; private JsonObject initialBrowseContinuationResponse; @@ -174,14 +177,10 @@ private JsonObject getPlaylistHeader() { @Nonnull @Override public String getName() throws ParsingException { - final String name = getTextFromObject(getPlaylistInfo().getObject("title")); - if (!isNullOrEmpty(name)) { - return name; - } - - return browseMetadataResponse.getObject(MICROFORMAT) - .getObject("microformatDataRenderer") - .getString("title"); + return getTextFromObject(getPlaylistInfo().getObject(TITLE)) + .orElseGet(() -> browseMetadataResponse.getObject(MICROFORMAT) + .getObject("microformatDataRenderer") + .getString(TITLE)); } @Nonnull @@ -191,13 +190,13 @@ public List getThumbnails() throws ParsingException { if (isNewPlaylistInterface) { playlistMetadataThumbnailsArray = getPlaylistHeader().getObject("playlistHeaderBanner") .getObject("heroPlaylistThumbnailRenderer") - .getObject("thumbnail") - .getArray("thumbnails"); + .getObject(THUMBNAIL) + .getArray(THUMBNAILS); } else { playlistMetadataThumbnailsArray = playlistInfo.getObject("thumbnailRenderer") .getObject("playlistVideoThumbnailRenderer") - .getObject("thumbnail") - .getArray("thumbnails"); + .getObject(THUMBNAIL) + .getArray(THUMBNAILS); } if (!isNullOrEmpty(playlistMetadataThumbnailsArray)) { @@ -207,8 +206,8 @@ public List getThumbnails() throws ParsingException { // This data structure is returned in both layouts final JsonArray microFormatThumbnailsArray = browseMetadataResponse.getObject(MICROFORMAT) .getObject("microformatDataRenderer") - .getObject("thumbnail") - .getArray("thumbnails"); + .getObject(THUMBNAIL) + .getArray(THUMBNAILS); if (!isNullOrEmpty(microFormatThumbnailsArray)) { return getImagesFromThumbnailsArray(microFormatThumbnailsArray); @@ -225,7 +224,7 @@ public String getUploaderUrl() throws ParsingException { .getArray("runs") .getObject(0) .getObject("navigationEndpoint") - : getUploaderInfo().getObject("navigationEndpoint")); + : getUploaderInfo().getObject("navigationEndpoint")).orElse(null); } catch (final Exception e) { throw new ParsingException("Could not get playlist uploader url", e); } @@ -233,13 +232,10 @@ public String getUploaderUrl() throws ParsingException { @Override public String getUploaderName() throws ParsingException { - try { - return getTextFromObject(isNewPlaylistInterface - ? getPlaylistHeader().getObject("ownerText") - : getUploaderInfo().getObject("title")); - } catch (final Exception e) { - throw new ParsingException("Could not get playlist uploader name", e); - } + final var jsonObject = isNewPlaylistInterface + ? getPlaylistHeader().getObject("ownerText") + : getUploaderInfo().getObject(TITLE); + return getTextFromObjectOrThrow(jsonObject, "playlist uploader name"); } @Nonnull @@ -251,8 +247,8 @@ public List getUploaderAvatars() throws ParsingException { } try { - return getImagesFromThumbnailsArray(getUploaderInfo().getObject("thumbnail") - .getArray("thumbnails")); + return getImagesFromThumbnailsArray(getUploaderInfo().getObject(THUMBNAIL) + .getArray(THUMBNAILS)); } catch (final Exception e) { throw new ParsingException("Could not get playlist uploader avatars", e); } @@ -266,60 +262,32 @@ public boolean isUploaderVerified() throws ParsingException { @Override public long getStreamCount() throws ParsingException { - if (isNewPlaylistInterface) { - final String numVideosText = - getTextFromObject(getPlaylistHeader().getObject("numVideosText")); - if (numVideosText != null) { - try { - return Long.parseLong(Utils.removeNonDigitCharacters(numVideosText)); - } catch (final NumberFormatException ignored) { - } - } - - final String firstByLineRendererText = getTextFromObject( - getPlaylistHeader().getArray("byline") - .getObject(0) - .getObject("text")); - - if (firstByLineRendererText != null) { - try { - return Long.parseLong(Utils.removeNonDigitCharacters(firstByLineRendererText)); - } catch (final NumberFormatException ignored) { - } - } - } - - // These data structures are returned in both layouts - final JsonArray briefStats = - (isNewPlaylistInterface ? getPlaylistHeader() : getPlaylistInfo()) - .getArray("briefStats"); - if (!briefStats.isEmpty()) { - final String briefsStatsText = getTextFromObject(briefStats.getObject(0)); - if (briefsStatsText != null) { - return Long.parseLong(Utils.removeNonDigitCharacters(briefsStatsText)); - } - } - - final JsonArray stats = (isNewPlaylistInterface ? getPlaylistHeader() : getPlaylistInfo()) - .getArray("stats"); - if (!stats.isEmpty()) { - final String statsText = getTextFromObject(stats.getObject(0)); - if (statsText != null) { - return Long.parseLong(Utils.removeNonDigitCharacters(statsText)); - } - } - - return ITEM_COUNT_UNKNOWN; + final var header = getPlaylistHeader(); + final Optional count = isNewPlaylistInterface + ? getTextFromObject(header.getObject("numVideosText")) + .or(() -> getTextFromObject(header.getArray("byline") + .getObject(0).getObject("text"))) + : Optional.empty(); + final var playlist = isNewPlaylistInterface ? header : getPlaylistInfo(); + + // "briefStats" and "stats" are returned in both layouts + return count.or(() -> getTextFromObject(playlist.getArray("briefStats").getObject(0))) + .or(() -> getTextFromObject(playlist.getArray("stats").getObject(0))) + .map(numText -> { + try { + return Long.parseLong(Utils.removeNonDigitCharacters(numText)); + } catch (final NumberFormatException e) { + return null; + } + }) + .orElse(ITEM_COUNT_UNKNOWN); } @Nonnull @Override public Description getDescription() throws ParsingException { - final String description = getTextFromObject( - getPlaylistInfo().getObject("description"), - true - ); - + final var descriptionObj = getPlaylistInfo().getObject("description"); + final String description = getTextFromObject(descriptionObj, true).orElse(null); return new Description(description, Description.HTML); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistInfoItemExtractor.java index a0584a20fb..5d093696c0 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistInfoItemExtractor.java @@ -12,9 +12,11 @@ import javax.annotation.Nonnull; import java.util.List; +import java.util.Optional; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getImagesFromThumbnailsArray; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObjectOrThrow; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromObject; public class YoutubePlaylistInfoItemExtractor implements PlaylistInfoItemExtractor { @@ -44,11 +46,7 @@ public List getThumbnails() throws ParsingException { @Override public String getName() throws ParsingException { - try { - return getTextFromObject(playlistInfoItem.getObject("title")); - } catch (final Exception e) { - throw new ParsingException("Could not get name", e); - } + return getTextFromObjectOrThrow(playlistInfoItem.getObject("title"), "name"); } @Override @@ -63,17 +61,14 @@ public String getUrl() throws ParsingException { @Override public String getUploaderName() throws ParsingException { - try { - return getTextFromObject(playlistInfoItem.getObject("longBylineText")); - } catch (final Exception e) { - throw new ParsingException("Could not get uploader name", e); - } + return getTextFromObjectOrThrow(playlistInfoItem.getObject("longBylineText"), + "uploader name"); } @Override public String getUploaderUrl() throws ParsingException { try { - return getUrlFromObject(playlistInfoItem.getObject("longBylineText")); + return getUrlFromObject(playlistInfoItem.getObject("longBylineText")).orElse(null); } catch (final Exception e) { throw new ParsingException("Could not get uploader url", e); } @@ -90,18 +85,10 @@ public boolean isUploaderVerified() throws ParsingException { @Override public long getStreamCount() throws ParsingException { - String videoCountText = playlistInfoItem.getString("videoCount"); - if (videoCountText == null) { - videoCountText = getTextFromObject(playlistInfoItem.getObject("videoCountText")); - } - - if (videoCountText == null) { - videoCountText = getTextFromObject(playlistInfoItem.getObject("videoCountShortText")); - } - - if (videoCountText == null) { - throw new ParsingException("Could not get stream count"); - } + final var videoCountText = Optional.ofNullable(playlistInfoItem.getString("videoCount")) + .or(() -> getTextFromObject(playlistInfoItem.getObject("videoCountText"))) + .or(() -> getTextFromObject(playlistInfoItem.getObject("videoCountShortText"))) + .orElseThrow(() -> new ParsingException("Could not get stream count")); try { return Long.parseLong(Utils.removeNonDigitCharacters(videoCountText)); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeReelInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeReelInfoItemExtractor.java index 71c7440b42..1dcb890183 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeReelInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeReelInfoItemExtractor.java @@ -1,8 +1,8 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObjectOrThrow; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getThumbnailsFromInfoItem; -import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; import com.grack.nanojson.JsonObject; @@ -45,7 +45,7 @@ public YoutubeReelInfoItemExtractor(@Nonnull final JsonObject reelInfo) { @Override public String getName() throws ParsingException { - return getTextFromObject(reelInfo.getObject("headline")); + return getTextFromObject(reelInfo.getObject("headline")).orElse(null); } @Override @@ -71,17 +71,13 @@ public StreamType getStreamType() throws ParsingException { @Override public long getViewCount() throws ParsingException { - final String viewCountText = getTextFromObject(reelInfo.getObject("viewCountText")); - if (!isNullOrEmpty(viewCountText)) { - // This approach is language dependent - if (viewCountText.toLowerCase().contains("no views")) { - return 0; - } - - return Utils.mixedNumberWordToLong(viewCountText); + final String viewCountText = getTextFromObjectOrThrow( + reelInfo.getObject("viewCountText"), "short view count"); + // This approach is language dependent + if (viewCountText.toLowerCase().contains("no views")) { + return 0; } - - throw new ParsingException("Could not get short view count"); + return Utils.mixedNumberWordToLong(viewCountText); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java index e888787cf9..ec12b8820c 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java @@ -35,7 +35,6 @@ import java.io.IOException; import java.nio.charset.StandardCharsets; import java.util.List; -import java.util.Objects; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -130,11 +129,11 @@ public String getSearchSuggestion() throws ParsingException { "correctedQueryEndpoint.searchEndpoint.query"); } - return Objects.requireNonNullElse( - getTextFromObject(itemSectionRenderer.getArray("contents") - .getObject(0) - .getObject("showingResultsForRenderer") - .getObject("correctedQuery")), ""); + final var query = itemSectionRenderer.getArray("contents") + .getObject(0) + .getObject("showingResultsForRenderer") + .getObject("correctedQuery"); + return getTextFromObject(query).orElse(""); } @Override @@ -231,7 +230,7 @@ private void collectStreamsFrom(final MultiInfoItemsCollector collector, if (item.has("backgroundPromoRenderer")) { throw new NothingFoundException( getTextFromObject(item.getObject("backgroundPromoRenderer") - .getObject("bodyText"))); + .getObject("bodyText")).orElse("")); } else if (item.has("videoRenderer") && extractVideoResults) { collector.commit(new YoutubeStreamInfoItemExtractor( item.getObject("videoRenderer"), timeAgoParser)); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeShowRendererInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeShowRendererInfoItemExtractor.java index c7119907e0..0c2f02c5d8 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeShowRendererInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeShowRendererInfoItemExtractor.java @@ -7,7 +7,6 @@ import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromObject; -import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; /** * A {@link YoutubeBaseShowInfoItemExtractor} implementation for {@code showRenderer}s. @@ -27,26 +26,16 @@ class YoutubeShowRendererInfoItemExtractor extends YoutubeBaseShowInfoItemExtrac @Override public String getUploaderName() throws ParsingException { - String name = getTextFromObject(longBylineText); - if (isNullOrEmpty(name)) { - name = getTextFromObject(shortBylineText); - if (isNullOrEmpty(name)) { - throw new ParsingException("Could not get uploader name"); - } - } - return name; + return getTextFromObject(longBylineText) + .or(() -> getTextFromObject(shortBylineText)) + .orElseThrow(() -> new ParsingException("Could not get uploader name")); } @Override public String getUploaderUrl() throws ParsingException { - String uploaderUrl = getUrlFromObject(longBylineText); - if (uploaderUrl == null) { - uploaderUrl = getUrlFromObject(shortBylineText); - if (uploaderUrl == null) { - throw new ParsingException("Could not get uploader URL"); - } - } - return uploaderUrl; + return getUrlFromObject(longBylineText) + .or(() -> getUrlFromObject(shortBylineText)) + .orElseThrow(() -> new ParsingException("Could not get uploader URL")); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index 1927f6dea2..a13656c7dc 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -26,13 +26,18 @@ import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.CONTENT_CHECK_OK; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.CPN; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.RACY_CHECK_OK; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.STRING_PREDICATE; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.VIDEO_ID; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.generateContentPlaybackNonce; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getFirstCollaborator; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getImagesFromThumbnailsArray; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.hasArtistOrVerifiedIconBadgeAttachment; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.SUBSCRIBER_COUNT_TEXT; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.TITLE; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; import com.grack.nanojson.JsonArray; @@ -53,8 +58,8 @@ import org.schabi.newpipe.extractor.exceptions.PaidContentException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.PrivateContentException; -import org.schabi.newpipe.extractor.exceptions.YoutubeMusicPremiumContentException; import org.schabi.newpipe.extractor.exceptions.SignInConfirmNotBotException; +import org.schabi.newpipe.extractor.exceptions.YoutubeMusicPremiumContentException; import org.schabi.newpipe.extractor.linkhandler.LinkHandler; import org.schabi.newpipe.extractor.localization.ContentCountry; import org.schabi.newpipe.extractor.localization.DateWrapper; @@ -118,7 +123,10 @@ public class YoutubeStreamExtractor extends StreamExtractor { private static final String THUMBNAIL = "thumbnail"; private static final String THUMBNAILS = "thumbnails"; private static final String VIDEO_DETAILS = "videoDetails"; - private static final String TITLE = "title"; + private static final String BADGES = "badges"; + private static final String VIEW_COUNT = "viewCount"; + private static final String ACCESSIBILITY_DATA = "accessibilityData"; + private static final String LABEL = "label"; @Nullable private static PoTokenProvider poTokenProvider; @@ -164,21 +172,10 @@ public YoutubeStreamExtractor(final StreamingService service, final LinkHandler @Override public String getName() throws ParsingException { assertPageFetched(); - String title; - // Try to get the video's original title, which is untranslated - title = playerResponse.getObject(VIDEO_DETAILS) - .getString(TITLE); - - if (isNullOrEmpty(title)) { - title = getTextFromObject(getVideoPrimaryInfoRenderer().getObject(TITLE)); - - if (isNullOrEmpty(title)) { - throw new ParsingException("Could not get name"); - } - } - - return title; + return Optional.ofNullable(playerResponse.getObject(VIDEO_DETAILS).getString(TITLE)) + .or(() -> getTextFromObject(getVideoPrimaryInfoRenderer().getObject(TITLE))) + .orElseThrow(() -> new ParsingException("Could not get name")); } @Nullable @@ -205,18 +202,19 @@ public String getTextualUploadDate() { return null; } - final var textObject = getVideoPrimaryInfoRenderer().getObject("dateText"); - final String rendererDateText = getTextFromObject(textObject); - if (rendererDateText == null) { - return null; - } else if (rendererDateText.startsWith(PREMIERED_ON)) { // Premiered on 21 Feb 2020 - return rendererDateText.substring(PREMIERED_ON.length()); - } else if (rendererDateText.startsWith(PREMIERED)) { - // Premiered 20 hours ago / Premiered Feb 21, 2020 - return rendererDateText.substring(PREMIERED.length()); - } else { - return rendererDateText; - } + return getTextFromObject(getVideoPrimaryInfoRenderer().getObject("dateText")) + .map(rendererDateText -> { + if (rendererDateText.startsWith(PREMIERED_ON)) { + // Premiered on 21 Feb 2020 + return rendererDateText.substring(PREMIERED_ON.length()); + } else if (rendererDateText.startsWith(PREMIERED)) { + // Premiered 20 hours ago / Premiered Feb 21, 2020 + return rendererDateText.substring(PREMIERED.length()); + } else { + return rendererDateText; + } + }) + .orElse(null); } @Override @@ -266,35 +264,24 @@ public List getThumbnails() throws ParsingException { @Nonnull @Override - public Description getDescription() throws ParsingException { + public Description getDescription() { assertPageFetched(); // Description with more info on links - final String videoSecondaryInfoRendererDescription = getTextFromObject( - getVideoSecondaryInfoRenderer().getObject("description"), - true); - if (!isNullOrEmpty(videoSecondaryInfoRendererDescription)) { - return new Description(videoSecondaryInfoRendererDescription, Description.HTML); - } - - final String attributedDescription = attributedDescriptionToHtml( - getVideoSecondaryInfoRenderer().getObject("attributedDescription")); - if (!isNullOrEmpty(attributedDescription)) { - return new Description(attributedDescription, Description.HTML); - } - - String description = playerResponse.getObject(VIDEO_DETAILS) - .getString("shortDescription"); - if (description == null) { - final JsonObject descriptionObject = playerMicroFormatRenderer.getObject("description"); - description = getTextFromObject(descriptionObject); - } - - // Raw non-html description - return new Description(description, Description.PLAIN_TEXT); + return getTextFromObject(getVideoSecondaryInfoRenderer().getObject("description"), true) + .or(() -> Optional.ofNullable(attributedDescriptionToHtml( + getVideoSecondaryInfoRenderer().getObject("attributedDescription")))) + .map(description -> new Description(description, Description.HTML)) + .or(() -> Optional.ofNullable(playerResponse.getObject(VIDEO_DETAILS) + .getString("shortDescription")) + .or(() -> getTextFromObject(playerMicroFormatRenderer + .getObject("description"))) + // Raw non-html description + .map(description -> new Description(description, Description.PLAIN_TEXT))) + .orElse(Description.EMPTY_DESCRIPTION); } @Override - public int getAgeLimit() throws ParsingException { + public int getAgeLimit() { if (ageLimit != -1) { return ageLimit; } @@ -370,17 +357,11 @@ public long getTimeStamp() throws ParsingException { @Override public long getViewCount() throws ParsingException { - String views = getTextFromObject(getVideoPrimaryInfoRenderer().getObject("viewCount") - .getObject("videoViewCountRenderer").getObject("viewCount")); - - if (isNullOrEmpty(views)) { - views = playerResponse.getObject(VIDEO_DETAILS) - .getString("viewCount"); - - if (isNullOrEmpty(views)) { - throw new ParsingException("Could not get view count"); - } - } + final var views = getTextFromObject(getVideoPrimaryInfoRenderer().getObject(VIEW_COUNT) + .getObject("videoViewCountRenderer").getObject(VIEW_COUNT)) + .or(() -> Optional.ofNullable(playerResponse.getObject(VIDEO_DETAILS) + .getString(VIEW_COUNT))) + .orElseThrow(() -> new ParsingException("Could not get view count")); if (views.toLowerCase().contains("no views")) { return 0; @@ -404,105 +385,79 @@ public long getLikeCount() throws ParsingException { .getObject("menuRenderer") .getArray("topLevelButtons"); - try { - return parseLikeCountFromLikeButtonViewModel(topLevelButtons); - } catch (final ParsingException ignored) { - // A segmentedLikeDislikeButtonRenderer could be returned instead of a - // segmentedLikeDislikeButtonViewModel, so ignore extraction errors relative to - // segmentedLikeDislikeButtonViewModel object - } - - try { - return parseLikeCountFromLikeButtonRenderer(topLevelButtons); - } catch (final ParsingException e) { - throw new ParsingException("Could not get like count", e); - } + return parseLikeCountFromLikeButtonViewModel(topLevelButtons) + // A segmentedLikeDislikeButtonRenderer could be returned instead of a + // segmentedLikeDislikeButtonViewModel, so ignore extraction errors relative to + // segmentedLikeDislikeButtonViewModel object + .or(() -> parseLikeCountFromLikeButtonRenderer(topLevelButtons)) + .orElseThrow(() -> new ParsingException("Could not get like count")); } - private static long parseLikeCountFromLikeButtonRenderer( - @Nonnull final JsonArray topLevelButtons) throws ParsingException { - String likesString = null; - final JsonObject likeToggleButtonRenderer = topLevelButtons.streamAsJsonObjects() + private static Optional parseLikeCountFromLikeButtonRenderer( + @Nonnull final JsonArray topLevelButtons) { + return topLevelButtons.streamAsJsonObjects() .map(button -> button.getObject("segmentedLikeDislikeButtonRenderer") .getObject("likeButton") - .getObject("toggleButtonRenderer")) - .filter(toggleButtonRenderer -> !isNullOrEmpty(toggleButtonRenderer)) + .getObject("toggleButtonRenderer", null)) + .filter(Objects::nonNull) .findFirst() - .orElse(null); - - if (likeToggleButtonRenderer != null) { - // Use one of the accessibility strings available (this one has the same path as the - // one used for comments' like count extraction) - likesString = likeToggleButtonRenderer.getObject("accessibilityData") - .getObject("accessibilityData") - .getString("label"); - - // Use the other accessibility string available which contains the exact like count - if (likesString == null) { - likesString = likeToggleButtonRenderer.getObject("accessibility") - .getString("label"); - } - - // Last method: use the defaultText's accessibility data, which contains the exact like - // count too, except when it is equal to 0, where a localized string is returned instead - if (likesString == null) { - likesString = likeToggleButtonRenderer.getObject("defaultText") - .getObject("accessibility") - .getObject("accessibilityData") - .getString("label"); - } - - // This check only works with English localizations! - if (likesString != null && likesString.toLowerCase().contains("no likes")) { - return 0; - } - } - - // If ratings are allowed and the likes string is null, it means that we couldn't extract - // the full like count from accessibility data - if (likesString == null) { - throw new ParsingException("Could not get like count from accessibility data"); - } - - try { - return Long.parseLong(Utils.removeNonDigitCharacters(likesString)); - } catch (final NumberFormatException e) { - throw new ParsingException("Could not parse \"" + likesString + "\" as a long", e); - } + .flatMap(toggleButtonRenderer -> { + // Use one of the accessibility strings available (this one has the same path + // as the one used for comments' like count extraction) + return Optional.ofNullable(toggleButtonRenderer.getObject(ACCESSIBILITY_DATA) + .getObject(ACCESSIBILITY_DATA) + .getString(LABEL)) + + // Use the other accessibility string available which contains the exact + // like count + .or(() -> Optional.ofNullable(toggleButtonRenderer + .getObject("accessibility").getString(LABEL))) + + // Last method: use the defaultText's accessibility data, which contains + // the exact like count too, except when it is equal to 0, where a + // localized string is returned instead + .or(() -> Optional.ofNullable( + toggleButtonRenderer.getObject("defaultText") + .getObject("accessibility") + .getObject(ACCESSIBILITY_DATA) + .getString(LABEL))); + }) + .map(likesString -> { + if (likesString.toLowerCase().contains("no likes")) { + return 0L; + } else { + try { + return Long.parseLong(Utils.removeNonDigitCharacters(likesString)); + } catch (final NumberFormatException e) { + return null; + } + } + }); } - private static long parseLikeCountFromLikeButtonViewModel( - @Nonnull final JsonArray topLevelButtons) throws ParsingException { + private static Optional parseLikeCountFromLikeButtonViewModel( + @Nonnull final JsonArray topLevelButtons) { // Try first with the current video actions buttons data structure - final JsonObject likeToggleButtonViewModel = topLevelButtons.streamAsJsonObjects() + return topLevelButtons.streamAsJsonObjects() .map(button -> button.getObject("segmentedLikeDislikeButtonViewModel") .getObject("likeButtonViewModel") .getObject("likeButtonViewModel") .getObject("toggleButtonViewModel") .getObject("toggleButtonViewModel") .getObject("defaultButtonViewModel") - .getObject("buttonViewModel")) - .filter(buttonViewModel -> !isNullOrEmpty(buttonViewModel)) + .getObject("buttonViewModel") + .getString("accessibilityText")) + .filter(Objects::nonNull) .findFirst() - .orElse(null); - - if (likeToggleButtonViewModel == null) { - throw new ParsingException("Could not find buttonViewModel object"); - } - - final String accessibilityText = likeToggleButtonViewModel.getString("accessibilityText"); - if (accessibilityText == null) { - throw new ParsingException("Could not find buttonViewModel's accessibilityText string"); - } - - // The like count is always returned as a number in this element, even for videos with no - // likes - try { - return Long.parseLong(Utils.removeNonDigitCharacters(accessibilityText)); - } catch (final NumberFormatException e) { - throw new ParsingException( - "Could not parse \"" + accessibilityText + "\" as a long", e); - } + .map(accessibilityText -> { + try { + // The like count is always returned as a number in this element, even for + // videos with no likes + return Long.parseLong(Utils.removeNonDigitCharacters(accessibilityText)); + } catch (final NumberFormatException e) { + return null; + } + }); } @Nonnull @@ -540,26 +495,19 @@ public String getUploaderName() throws ParsingException { } @Override - public boolean isUploaderVerified() throws ParsingException { - final JsonObject videoOwnerRenderer = getVideoSecondaryInfoRenderer() - .getObject("owner") - .getObject("videoOwnerRenderer"); - - if (videoOwnerRenderer.has("badges")) { - return YoutubeParsingHelper.isVerified(videoOwnerRenderer - .getArray("badges")); - } - - - final JsonObject channel = YoutubeParsingHelper.getFirstCollaborator( - videoOwnerRenderer.getObject("navigationEndpoint")); - if (channel == null) { - return false; - } - - return YoutubeParsingHelper.hasArtistOrVerifiedIconBadgeAttachment( - channel.getObject(TITLE) - .getArray("attachmentRuns")); + public boolean isUploaderVerified() { + final var videoOwnerRenderer = getVideoSecondaryInfoRenderer() + .getObject("owner") + .getObject("videoOwnerRenderer"); + + return Optional.ofNullable(videoOwnerRenderer.getArray(BADGES, null)) + .map(YoutubeParsingHelper::isVerified) + .or(() -> getFirstCollaborator(videoOwnerRenderer.getObject("navigationEndpoint")) + .map(channel -> { + final var runs = channel.getObject(TITLE).getArray("attachmentRuns"); + return hasArtistOrVerifiedIconBadgeAttachment(runs); + })) + .orElse(false); } @Nonnull @@ -593,26 +541,19 @@ public List getUploaderAvatars() throws ParsingException { @Override public long getUploaderSubscriberCount() throws ParsingException { - final JsonObject videoOwnerRenderer = JsonUtils.getObject(videoSecondaryInfoRenderer, + final var renderer = JsonUtils.getObject(getVideoSecondaryInfoRenderer(), "owner.videoOwnerRenderer"); + final var subscriberCountText = getTextFromObject(renderer.getObject(SUBSCRIBER_COUNT_TEXT)) + .or(() -> getFirstCollaborator(renderer.getObject("navigationEndpoint")) + .map(endpoint -> endpoint.getObject("subtitle").getString("content"))) + .filter(STRING_PREDICATE); - String subscriberCountText = null; - if (videoOwnerRenderer.has("subscriberCountText")) { - subscriberCountText = getTextFromObject(videoOwnerRenderer - .getObject("subscriberCountText")); - } else { - final String content = YoutubeParsingHelper.getFirstCollaborator( - videoOwnerRenderer.getObject("navigationEndpoint") - ).getObject("subtitle").getString("content"); - subscriberCountText = content.split("•")[1]; - } - - if (isNullOrEmpty(subscriberCountText)) { + if (subscriberCountText.isEmpty()) { return UNKNOWN_SUBSCRIBER_COUNT; } try { - return Utils.mixedNumberWordToLong(subscriberCountText); + return Utils.mixedNumberWordToLong(subscriberCountText.get()); } catch (final NumberFormatException e) { throw new ParsingException("Could not get uploader subscriber count", e); } @@ -620,7 +561,7 @@ public long getUploaderSubscriberCount() throws ParsingException { @Nonnull @Override - public String getDashMpdUrl() throws ParsingException { + public String getDashMpdUrl() { assertPageFetched(); // There is no DASH manifest available with the iOS client @@ -634,7 +575,7 @@ public String getDashMpdUrl() throws ParsingException { @Nonnull @Override - public String getHlsUrl() throws ParsingException { + public String getHlsUrl() { assertPageFetched(); // Return HLS manifest of the iOS client first because on livestreams, the HLS manifest @@ -816,13 +757,9 @@ public MultiInfoItemsCollector getRelatedItems() throws ExtractionException { */ @Override public String getErrorMessage() { - try { - return getTextFromObject(playerResponse.getObject(PLAYABILITY_STATUS) - .getObject("errorScreen").getObject("playerErrorMessageRenderer") - .getObject("reason")); - } catch (final NullPointerException e) { - return null; // No error message - } + return getTextFromObject(playerResponse.getObject(PLAYABILITY_STATUS) + .getObject("errorScreen").getObject("playerErrorMessageRenderer") + .getObject("reason")).orElse(null); } /*////////////////////////////////////////////////////////////////////////// @@ -1471,7 +1408,7 @@ public List getFrames() throws ExtractionException { @Override public Privacy getPrivacy() { return playerMicroFormatRenderer.getBoolean("isUnlisted") - || getVideoPrimaryInfoRenderer().getArray("badges") + || getVideoPrimaryInfoRenderer().getArray(BADGES) .streamAsJsonObjects() .anyMatch(badge -> "PRIVACY_UNLISTED".equals(badge.getObject("metadataBadgeRenderer") @@ -1489,20 +1426,18 @@ public String getCategory() { @Nonnull @Override - public String getLicence() throws ParsingException { - final JsonObject metadataRowRenderer = getVideoSecondaryInfoRenderer() + public String getLicence() { + final var metadataRowRenderer = getVideoSecondaryInfoRenderer() .getObject("metadataRowContainer") .getObject("metadataRowContainerRenderer") .getArray("rows") .getObject(0) .getObject("metadataRowRenderer"); - - final JsonArray contents = metadataRowRenderer.getArray("contents"); - final String license = getTextFromObject(contents.getObject(0)); - return license != null - && "Licence".equals(getTextFromObject(metadataRowRenderer.getObject(TITLE))) - ? license - : "YouTube licence"; + final var contents = metadataRowRenderer.getArray("contents"); + final var title = getTextFromObject(metadataRowRenderer.getObject(TITLE)).orElse(null); + return getTextFromObject(contents.getObject(0)) + .filter(license -> "Licence".equals(title)) + .orElse("YouTube licence"); } @Override @@ -1564,10 +1499,8 @@ public List getStreamSegments() throws ParsingException { break; } - final String title = getTextFromObject(segmentJson.getObject(TITLE)); - if (isNullOrEmpty(title)) { - throw new ParsingException("Could not get stream segment title."); - } + final String title = getTextFromObject(segmentJson.getObject(TITLE)) + .orElseThrow(() -> new ParsingException("Could not get stream segment title.")); final StreamSegment segment = new StreamSegment(title, startTimeSeconds); segment.setUrl(getUrl() + "?t=" + startTimeSeconds); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java index a87358fe68..2721930659 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java @@ -18,10 +18,10 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.NAVIGATION_ENDPOINT; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getThumbnailsFromInfoItem; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getImagesFromThumbnailsArray; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; import com.grack.nanojson.JsonArray; @@ -33,9 +33,9 @@ import org.schabi.newpipe.extractor.localization.TimeAgoParser; import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeStreamLinkHandlerFactory; -import org.schabi.newpipe.extractor.stream.ContentAvailability; import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor; import org.schabi.newpipe.extractor.stream.StreamType; +import org.schabi.newpipe.extractor.stream.ContentAvailability; import org.schabi.newpipe.extractor.utils.JsonUtils; import org.schabi.newpipe.extractor.utils.Parser; import org.schabi.newpipe.extractor.utils.Utils; @@ -48,14 +48,15 @@ import java.time.ZoneId; import java.time.format.DateTimeFormatter; import java.util.List; +import java.util.Optional; import java.util.regex.Pattern; -import java.util.stream.Collectors; public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor { private static final Pattern ACCESSIBILITY_DATA_VIEW_COUNT_REGEX = Pattern.compile("([\\d,]+) views$"); private static final String NO_VIEWS_LOWERCASE = "no views"; + private static final String VIDEO_INFO = "videoInfo"; private final JsonObject videoInfo; private final TimeAgoParser timeAgoParser; @@ -132,15 +133,9 @@ public String getUrl() throws ParsingException { @Override public String getName() throws ParsingException { final JsonObject title = videoInfo.getObject("title"); - final String name = getTextFromObject(title); - if (!isNullOrEmpty(name)) { - return name; - } - // Videos can have no title, e.g. https://www.youtube.com/watch?v=nc1kN8ZSfGQ - if (!isNullOrEmpty(title) && !title.has("runs")) { - return ""; - } - throw new ParsingException("Could not get name"); + return getTextFromObject(title) + .or(() -> Optional.ofNullable(!title.has("runs") ? "" : null)) + .orElseThrow(() -> new ParsingException("Could not get name")); } @Override @@ -149,85 +144,51 @@ public long getDuration() throws ParsingException { return -1; } - String duration = getTextFromObject(videoInfo.getObject("lengthText")); - - if (isNullOrEmpty(duration)) { - // Available in playlists for videos - duration = videoInfo.getString("lengthSeconds"); - - if (isNullOrEmpty(duration)) { - final List timeOverlays = videoInfo.getArray("thumbnailOverlays") + return getTextFromObject(videoInfo.getObject("lengthText")) + .or(() -> Optional.ofNullable(videoInfo.getString("lengthSeconds"))) + .or(() -> videoInfo.getArray("thumbnailOverlays") .streamAsJsonObjects() - .filter(thumbnailOverlay -> - thumbnailOverlay.has("thumbnailOverlayTimeStatusRenderer")) - .map(thumbnailOverlay -> getTextFromObject( - thumbnailOverlay.getObject("thumbnailOverlayTimeStatusRenderer") - .getObject("text"))) + .map(thumbnailOverlay -> { + final var thumbnailRendererText = thumbnailOverlay + .getObject("thumbnailOverlayTimeStatusRenderer") + .getObject("text"); + return getTextFromObject(thumbnailRendererText).orElse(null); + }) .filter(text -> !isNullOrEmpty(text)) - .collect(Collectors.toList()); - - for (final String timeOverlayText : timeOverlays) { + .findFirst()) + .map(duration -> { try { - return YoutubeParsingHelper.parseDurationString(timeOverlayText); - } catch (final ParsingException ex) { - // try next + return YoutubeParsingHelper.parseDurationString(duration); + } catch (final ParsingException e) { + return null; } - } - } - - if (isNullOrEmpty(duration)) { - if (isPremiere()) { - // Premieres can be livestreams, so the duration is not available in this - // case - return -1; - } - - throw new ParsingException("Could not get duration"); - } - } - - return YoutubeParsingHelper.parseDurationString(duration); + }) + // Premieres can be livestreams, so the duration is not available in this case + .or(() -> Optional.ofNullable(isPremiere() ? -1 : null)) + .orElseThrow(() -> new ParsingException("Could not get duration")); } @Override public String getUploaderName() throws ParsingException { - String name = getTextFromObject(videoInfo.getObject("longBylineText")); - - if (isNullOrEmpty(name)) { - name = getTextFromObject(videoInfo.getObject("ownerText")); - - if (isNullOrEmpty(name)) { - name = getTextFromObject(videoInfo.getObject("shortBylineText")); - - if (isNullOrEmpty(name)) { - throw new ParsingException("Could not get uploader name"); - } - } - } - - return name; + return getTextFromObject(videoInfo.getObject("longBylineText")) + .or(() -> getTextFromObject(videoInfo.getObject("ownerText"))) + .or(() -> getTextFromObject(videoInfo.getObject("shortBylineText"))) + .orElseThrow(() -> new ParsingException("Could not get uploader name")); } @Override public String getUploaderUrl() throws ParsingException { - String url = getUrlFromNavigationEndpoint(videoInfo.getObject("longBylineText") - .getArray("runs").getObject(0).getObject("navigationEndpoint")); - - if (isNullOrEmpty(url)) { - url = getUrlFromNavigationEndpoint(videoInfo.getObject("ownerText") - .getArray("runs").getObject(0).getObject("navigationEndpoint")); - - if (isNullOrEmpty(url)) { - url = getUrlFromNavigationEndpoint(videoInfo.getObject("shortBylineText") - .getArray("runs").getObject(0).getObject("navigationEndpoint")); - - if (isNullOrEmpty(url)) { - throw new ParsingException("Could not get uploader url"); - } - } - } + return getUrlFromNavigationEndpoint(videoInfo.getObject("longBylineText")) + .or(() -> getUrlFromNavigationEndpoint(videoInfo.getObject("ownerText"))) + .or(() -> getUrlFromNavigationEndpoint(videoInfo.getObject("shortBylineText"))) + .orElseThrow(() -> new ParsingException("Could not get uploader url")); + } - return url; + @Nonnull + private Optional getUrlFromNavigationEndpoint(@Nonnull final JsonObject jsonObject) { + final var endpoint = jsonObject.getArray("runs").getObject(0) + .getObject(NAVIGATION_ENDPOINT); + return YoutubeParsingHelper.getUrlFromNavigationEndpoint(endpoint); } @Nonnull @@ -266,19 +227,15 @@ public String getTextualUploadDate() throws ParsingException { return DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm").format(localDateTime); } - String publishedTimeText = getTextFromObject(videoInfo.getObject("publishedTimeText")); - - if (isNullOrEmpty(publishedTimeText) && videoInfo.has("videoInfo")) { - /* - Returned in playlists, in the form: view count separator upload date - */ - publishedTimeText = videoInfo.getObject("videoInfo") - .getArray("runs") - .getObject(2) - .getString("text"); - } - - return isNullOrEmpty(publishedTimeText) ? null : publishedTimeText; + return getTextFromObject(videoInfo.getObject("publishedTimeText")) + .or(() -> { + // Returned in playlists, in the form: view count separator upload date + return Optional.ofNullable(videoInfo.getObject(VIDEO_INFO) + .getArray("runs") + .getObject(2) + .getString("text")); + }) + .orElse(null); } @Nullable @@ -312,7 +269,8 @@ public long getViewCount() throws ParsingException { // Ignore all exceptions, as the view count can be hidden by creators, and so cannot be // found in this case - final String viewCountText = getTextFromObject(videoInfo.getObject("viewCountText")); + final String viewCountText = getTextFromObject(videoInfo.getObject("viewCountText")) + .orElse(null); if (!isNullOrEmpty(viewCountText)) { try { return getViewCountFromViewCountText(viewCountText, false); @@ -331,10 +289,10 @@ public long getViewCount() throws ParsingException { } // Fallback to a short view count, always used for livestreams (see why above) - if (videoInfo.has("videoInfo")) { + if (videoInfo.has(VIDEO_INFO)) { // Returned in playlists, in the form: view count separator upload date try { - return getViewCountFromViewCountText(videoInfo.getObject("videoInfo") + return getViewCountFromViewCountText(videoInfo.getObject(VIDEO_INFO) .getArray("runs") .getObject(0) .getString("text", ""), true); @@ -346,7 +304,8 @@ public long getViewCount() throws ParsingException { // Returned everywhere but in playlists, used by the website to show view counts try { final String shortViewCountText = - getTextFromObject(videoInfo.getObject("shortViewCountText")); + getTextFromObject(videoInfo.getObject("shortViewCountText")) + .orElse(null); if (!isNullOrEmpty(shortViewCountText)) { return getViewCountFromViewCountText(shortViewCountText, true); } @@ -427,24 +386,18 @@ private Instant getInstantFromPremiere() throws ParsingException { @Nullable @Override - public String getShortDescription() throws ParsingException { - if (videoInfo.has("detailedMetadataSnippets")) { - return getTextFromObject(videoInfo.getArray("detailedMetadataSnippets") - .getObject(0) - .getObject("snippetText")); - } - - if (videoInfo.has("descriptionSnippet")) { - return getTextFromObject(videoInfo.getObject("descriptionSnippet")); - } - - return null; + public String getShortDescription() { + return getTextFromObject(videoInfo.getArray("detailedMetadataSnippets") + .getObject(0) + .getObject("snippetText")) + .or(() -> getTextFromObject(videoInfo.getObject("descriptionSnippet"))) + .orElse(null); } @Override public boolean isShortFormContent() throws ParsingException { try { - final String webPageType = videoInfo.getObject("navigationEndpoint") + final String webPageType = videoInfo.getObject(NAVIGATION_ENDPOINT) .getObject("commandMetadata").getObject("webCommandMetadata") .getString("webPageType"); @@ -452,7 +405,7 @@ public boolean isShortFormContent() throws ParsingException { && webPageType.equals("WEB_PAGE_TYPE_SHORTS"); if (!isShort) { - isShort = videoInfo.getObject("navigationEndpoint").has("reelWatchEndpoint"); + isShort = videoInfo.getObject(NAVIGATION_ENDPOINT).has("reelWatchEndpoint"); } if (!isShort) { @@ -487,7 +440,7 @@ private boolean isMembersOnly() { @Nonnull @Override - public ContentAvailability getContentAvailability() throws ParsingException { + public ContentAvailability getContentAvailability() { if (isPremiere()) { return ContentAvailability.UPCOMING; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/kiosk/YoutubeTrendingExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/kiosk/YoutubeTrendingExtractor.java index 6ec1d98cc2..d1193f5c88 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/kiosk/YoutubeTrendingExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/kiosk/YoutubeTrendingExtractor.java @@ -23,7 +23,7 @@ import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextAtKey; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder; -import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.TITLE; import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonWriter; @@ -84,19 +84,10 @@ public InfoItemsPage getPage(final Page page) { @Override public String getName() throws ParsingException { final JsonObject header = initialData.getObject("header"); - String name = null; - if (header.has("feedTabbedHeaderRenderer")) { - name = getTextAtKey(header.getObject("feedTabbedHeaderRenderer"), "title"); - } else if (header.has("c4TabbedHeaderRenderer")) { - name = getTextAtKey(header.getObject("c4TabbedHeaderRenderer"), "title"); - } else if (header.has("pageHeaderRenderer")) { - name = getTextAtKey(header.getObject("pageHeaderRenderer"), "pageTitle"); - } - - if (isNullOrEmpty(name)) { - throw new ParsingException("Could not get Trending name"); - } - return name; + return getTextAtKey(header.getObject("feedTabbedHeaderRenderer"), TITLE) + .or(() -> getTextAtKey(header.getObject("c4TabbedHeaderRenderer"), TITLE)) + .or(() -> getTextAtKey(header.getObject("pageHeaderRenderer"), "pageTitle")) + .orElseThrow(() -> new ParsingException("Could not get Trending name")); } @Nonnull @@ -136,7 +127,7 @@ public InfoItemsPage getInitialPage() throws ParsingException { } else { // Filter Trending shorts and Recently trending sections which have a title, // contrary to normal trends - items = shelves.filter(shelfRenderer -> !shelfRenderer.has("title")); + items = shelves.filter(shelfRenderer -> !shelfRenderer.has(TITLE)); } items.flatMap(shelfRenderer -> shelfRenderer.getObject("content")