mirror of
https://github.com/TeamNewPipe/NewPipeExtractor.git
synced 2024-12-12 21:30:33 +05:30
Merge pull request #14 from mauriciocolli/refactor-extractor
Fix dash parser and more refactor
This commit is contained in:
commit
b5b25a4188
@ -14,7 +14,7 @@ public abstract class Extractor implements Serializable {
|
||||
this.urlIdHandler = urlIdHandler;
|
||||
this.serviceId = serviceId;
|
||||
this.url = url;
|
||||
this.previewInfoCollector = new StreamInfoItemCollector(urlIdHandler, serviceId);
|
||||
this.previewInfoCollector = new StreamInfoItemCollector(serviceId);
|
||||
}
|
||||
|
||||
public String getUrl() {
|
||||
|
@ -11,9 +11,9 @@ public abstract class Info implements Serializable {
|
||||
* Id of this Info object <br>
|
||||
* e.g. Youtube: https://www.youtube.com/watch?v=RER5qCTzZ7 > RER5qCTzZ7
|
||||
*/
|
||||
public String id = "";
|
||||
public String url = "";
|
||||
public String name = "";
|
||||
public String id;
|
||||
public String url;
|
||||
public String name;
|
||||
|
||||
public List<Throwable> errors = new Vector<>();
|
||||
}
|
||||
|
@ -1,7 +1,5 @@
|
||||
package org.schabi.newpipe.extractor;
|
||||
|
||||
import java.io.Serializable;
|
||||
|
||||
/*
|
||||
* Created by the-scrabi on 11.02.17.
|
||||
*
|
||||
@ -22,14 +20,21 @@ import java.io.Serializable;
|
||||
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
public interface InfoItem extends Serializable {
|
||||
enum InfoType {
|
||||
import java.io.Serializable;
|
||||
|
||||
public abstract class InfoItem implements Serializable {
|
||||
public enum InfoType {
|
||||
STREAM,
|
||||
PLAYLIST,
|
||||
CHANNEL
|
||||
}
|
||||
|
||||
InfoType infoType();
|
||||
String getTitle();
|
||||
String getLink();
|
||||
public InfoItem(InfoType infoType) {
|
||||
this.info_type = infoType;
|
||||
}
|
||||
|
||||
public final InfoType info_type;
|
||||
public int service_id = -1;
|
||||
public String url;
|
||||
public String name;
|
||||
}
|
||||
|
@ -25,7 +25,7 @@ import java.util.Vector;
|
||||
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
public class InfoItemCollector {
|
||||
public abstract class InfoItemCollector {
|
||||
private List<InfoItem> itemList = new Vector<>();
|
||||
private List<Throwable> errors = new Vector<>();
|
||||
private int serviceId = -1;
|
||||
|
@ -91,4 +91,17 @@ public enum MediaFormat {
|
||||
}
|
||||
return "";
|
||||
}
|
||||
|
||||
/**
|
||||
* Return the MediaFormat with the supplied mime type
|
||||
*
|
||||
* @return MediaFormat associated with this mime type,
|
||||
* or null if none match it.
|
||||
*/
|
||||
public static MediaFormat getFromMimeType(String mimeType) {
|
||||
for (MediaFormat vf : MediaFormat.values()) {
|
||||
if (vf.mimeType.equals(mimeType)) return vf;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
@ -1,6 +1,5 @@
|
||||
package org.schabi.newpipe.extractor.channel;
|
||||
|
||||
import org.schabi.newpipe.extractor.Extractor;
|
||||
import org.schabi.newpipe.extractor.ListExtractor;
|
||||
import org.schabi.newpipe.extractor.UrlIdHandler;
|
||||
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
|
||||
|
@ -29,21 +29,16 @@ import java.util.List;
|
||||
|
||||
public class ChannelInfo extends Info {
|
||||
|
||||
public static ChannelInfo getInfo(ChannelExtractor extractor)
|
||||
throws ParsingException {
|
||||
public static ChannelInfo getInfo(ChannelExtractor extractor) throws ParsingException {
|
||||
ChannelInfo info = new ChannelInfo();
|
||||
|
||||
// important data
|
||||
info.service_id = extractor.getServiceId();
|
||||
info.url = extractor.getUrl();
|
||||
info.id = extractor.getChannelId();
|
||||
info.name = extractor.getChannelName();
|
||||
info.hasMoreStreams = extractor.hasMoreStreams();
|
||||
info.has_more_streams = extractor.hasMoreStreams();
|
||||
|
||||
try {
|
||||
info.id = extractor.getChannelId();
|
||||
} catch (Exception e) {
|
||||
info.errors.add(e);
|
||||
}
|
||||
try {
|
||||
info.avatar_url = extractor.getAvatarUrl();
|
||||
} catch (Exception e) {
|
||||
@ -75,10 +70,10 @@ public class ChannelInfo extends Info {
|
||||
return info;
|
||||
}
|
||||
|
||||
public String avatar_url = "";
|
||||
public String banner_url = "";
|
||||
public String feed_url = "";
|
||||
public List<InfoItem> related_streams = null;
|
||||
public String avatar_url;
|
||||
public String banner_url;
|
||||
public String feed_url;
|
||||
public List<InfoItem> related_streams;
|
||||
public long subscriber_count = -1;
|
||||
public boolean hasMoreStreams = false;
|
||||
public boolean has_more_streams = false;
|
||||
}
|
||||
|
@ -22,25 +22,14 @@ import org.schabi.newpipe.extractor.InfoItem;
|
||||
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
public class ChannelInfoItem implements InfoItem {
|
||||
public class ChannelInfoItem extends InfoItem {
|
||||
|
||||
public int serviceId = -1;
|
||||
public String channelName = "";
|
||||
public String thumbnailUrl = "";
|
||||
public String webPageUrl = "";
|
||||
public String description = "";
|
||||
public long subscriberCount = -1;
|
||||
public long viewCount = -1;
|
||||
public String thumbnail_url;
|
||||
public String description;
|
||||
public long subscriber_count = -1;
|
||||
public long view_count = -1;
|
||||
|
||||
public InfoType infoType() {
|
||||
return InfoType.CHANNEL;
|
||||
}
|
||||
|
||||
public String getTitle() {
|
||||
return channelName;
|
||||
}
|
||||
|
||||
public String getLink() {
|
||||
return webPageUrl;
|
||||
public ChannelInfoItem() {
|
||||
super(InfoType.CHANNEL);
|
||||
}
|
||||
}
|
||||
|
@ -31,24 +31,24 @@ public class ChannelInfoItemCollector extends InfoItemCollector {
|
||||
public ChannelInfoItem extract(ChannelInfoItemExtractor extractor) throws ParsingException {
|
||||
ChannelInfoItem resultItem = new ChannelInfoItem();
|
||||
// important information
|
||||
resultItem.channelName = extractor.getChannelName();
|
||||
resultItem.name = extractor.getChannelName();
|
||||
|
||||
resultItem.serviceId = getServiceId();
|
||||
resultItem.webPageUrl = extractor.getWebPageUrl();
|
||||
resultItem.service_id = getServiceId();
|
||||
resultItem.url = extractor.getWebPageUrl();
|
||||
|
||||
// optional information
|
||||
try {
|
||||
resultItem.subscriberCount = extractor.getSubscriberCount();
|
||||
resultItem.subscriber_count = extractor.getSubscriberCount();
|
||||
} catch (Exception e) {
|
||||
addError(e);
|
||||
}
|
||||
try {
|
||||
resultItem.viewCount = extractor.getViewCount();
|
||||
resultItem.view_count = extractor.getViewCount();
|
||||
} catch (Exception e) {
|
||||
addError(e);
|
||||
}
|
||||
try {
|
||||
resultItem.thumbnailUrl = extractor.getThumbnailUrl();
|
||||
resultItem.thumbnail_url = extractor.getThumbnailUrl();
|
||||
} catch (Exception e) {
|
||||
addError(e);
|
||||
}
|
||||
|
11
exceptions/ContentNotAvailableException.java
Normal file
11
exceptions/ContentNotAvailableException.java
Normal file
@ -0,0 +1,11 @@
|
||||
package org.schabi.newpipe.extractor.exceptions;
|
||||
|
||||
public class ContentNotAvailableException extends ParsingException {
|
||||
public ContentNotAvailableException(String message) {
|
||||
super(message);
|
||||
}
|
||||
|
||||
public ContentNotAvailableException(String message, Throwable cause) {
|
||||
super(message, cause);
|
||||
}
|
||||
}
|
@ -1,6 +1,5 @@
|
||||
package org.schabi.newpipe.extractor.playlist;
|
||||
|
||||
import org.schabi.newpipe.extractor.Extractor;
|
||||
import org.schabi.newpipe.extractor.ListExtractor;
|
||||
import org.schabi.newpipe.extractor.UrlIdHandler;
|
||||
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
|
||||
|
@ -14,14 +14,10 @@ public class PlaylistInfo extends Info {
|
||||
|
||||
info.service_id = extractor.getServiceId();
|
||||
info.url = extractor.getUrl();
|
||||
info.id = extractor.getPlaylistId();
|
||||
info.name = extractor.getPlaylistName();
|
||||
info.hasMoreStreams = extractor.hasMoreStreams();
|
||||
info.has_more_streams = extractor.hasMoreStreams();
|
||||
|
||||
try {
|
||||
info.id = extractor.getPlaylistId();
|
||||
} catch (Exception e) {
|
||||
info.errors.add(e);
|
||||
}
|
||||
try {
|
||||
info.streams_count = extractor.getStreamsCount();
|
||||
} catch (Exception e) {
|
||||
@ -63,12 +59,12 @@ public class PlaylistInfo extends Info {
|
||||
return info;
|
||||
}
|
||||
|
||||
public String avatar_url = "";
|
||||
public String banner_url = "";
|
||||
public String uploader_url = "";
|
||||
public String uploader_name = "";
|
||||
public String uploader_avatar_url = "";
|
||||
public String avatar_url;
|
||||
public String banner_url;
|
||||
public String uploader_url;
|
||||
public String uploader_name;
|
||||
public String uploader_avatar_url;
|
||||
public long streams_count = 0;
|
||||
public List<InfoItem> related_streams = null;
|
||||
public boolean hasMoreStreams = false;
|
||||
public List<InfoItem> related_streams;
|
||||
public boolean has_more_streams;
|
||||
}
|
||||
|
@ -2,22 +2,15 @@ package org.schabi.newpipe.extractor.playlist;
|
||||
|
||||
import org.schabi.newpipe.extractor.InfoItem;
|
||||
|
||||
public class PlaylistInfoItem implements InfoItem {
|
||||
public class PlaylistInfoItem extends InfoItem {
|
||||
|
||||
public int serviceId = -1;
|
||||
public String name = "";
|
||||
public String thumbnailUrl = "";
|
||||
public String webPageUrl = "";
|
||||
public String thumbnail_url;
|
||||
/**
|
||||
* How many streams this playlist have
|
||||
*/
|
||||
public long streams_count = 0;
|
||||
|
||||
public InfoType infoType() {
|
||||
return InfoType.PLAYLIST;
|
||||
}
|
||||
|
||||
public String getTitle() {
|
||||
return name;
|
||||
}
|
||||
|
||||
public String getLink() {
|
||||
return webPageUrl;
|
||||
public PlaylistInfoItem() {
|
||||
super(InfoType.PLAYLIST);
|
||||
}
|
||||
}
|
||||
|
@ -1,6 +1,7 @@
|
||||
package org.schabi.newpipe.extractor.playlist;
|
||||
|
||||
import org.schabi.newpipe.extractor.InfoItemCollector;
|
||||
import org.schabi.newpipe.extractor.UrlIdHandler;
|
||||
import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
||||
|
||||
public class PlaylistInfoItemCollector extends InfoItemCollector {
|
||||
@ -12,10 +13,16 @@ public class PlaylistInfoItemCollector extends InfoItemCollector {
|
||||
final PlaylistInfoItem resultItem = new PlaylistInfoItem();
|
||||
|
||||
resultItem.name = extractor.getPlaylistName();
|
||||
resultItem.serviceId = getServiceId();
|
||||
resultItem.webPageUrl = extractor.getWebPageUrl();
|
||||
resultItem.service_id = getServiceId();
|
||||
resultItem.url = extractor.getWebPageUrl();
|
||||
|
||||
try {
|
||||
resultItem.thumbnailUrl = extractor.getThumbnailUrl();
|
||||
resultItem.thumbnail_url = extractor.getThumbnailUrl();
|
||||
} catch (Exception e) {
|
||||
addError(e);
|
||||
}
|
||||
try {
|
||||
resultItem.streams_count = extractor.getStreamsCount();
|
||||
} catch (Exception e) {
|
||||
addError(e);
|
||||
}
|
||||
|
@ -6,4 +6,5 @@ public interface PlaylistInfoItemExtractor {
|
||||
String getThumbnailUrl() throws ParsingException;
|
||||
String getPlaylistName() throws ParsingException;
|
||||
String getWebPageUrl() throws ParsingException;
|
||||
long getStreamsCount() throws ParsingException;
|
||||
}
|
||||
|
@ -1,7 +1,6 @@
|
||||
package org.schabi.newpipe.extractor.search;
|
||||
|
||||
import org.schabi.newpipe.extractor.InfoItemCollector;
|
||||
import org.schabi.newpipe.extractor.UrlIdHandler;
|
||||
import org.schabi.newpipe.extractor.channel.ChannelInfoItemCollector;
|
||||
import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor;
|
||||
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
|
||||
@ -30,15 +29,15 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor;
|
||||
*/
|
||||
|
||||
public class InfoItemSearchCollector extends InfoItemCollector {
|
||||
private String suggestion = "";
|
||||
private String suggestion;
|
||||
private StreamInfoItemCollector streamCollector;
|
||||
private ChannelInfoItemCollector channelCollector;
|
||||
|
||||
SearchResult result = new SearchResult();
|
||||
private SearchResult result = new SearchResult();
|
||||
|
||||
InfoItemSearchCollector(UrlIdHandler handler, int serviceId) {
|
||||
InfoItemSearchCollector(int serviceId) {
|
||||
super(serviceId);
|
||||
streamCollector = new StreamInfoItemCollector(handler, serviceId);
|
||||
streamCollector = new StreamInfoItemCollector(serviceId);
|
||||
channelCollector = new ChannelInfoItemCollector(serviceId);
|
||||
}
|
||||
|
||||
|
@ -1,6 +1,5 @@
|
||||
package org.schabi.newpipe.extractor.search;
|
||||
|
||||
import org.schabi.newpipe.extractor.UrlIdHandler;
|
||||
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
|
||||
|
||||
import java.io.IOException;
|
||||
@ -28,7 +27,7 @@ import java.util.EnumSet;
|
||||
|
||||
public abstract class SearchEngine {
|
||||
public enum Filter {
|
||||
STREAM, CHANNEL, PLAY_LIST
|
||||
STREAM, CHANNEL, PLAYLIST
|
||||
}
|
||||
|
||||
public static class NothingFoundException extends ExtractionException {
|
||||
@ -39,8 +38,8 @@ public abstract class SearchEngine {
|
||||
|
||||
private InfoItemSearchCollector collector;
|
||||
|
||||
public SearchEngine(UrlIdHandler urlIdHandler, int serviceId) {
|
||||
collector = new InfoItemSearchCollector(urlIdHandler, serviceId);
|
||||
public SearchEngine(int serviceId) {
|
||||
collector = new InfoItemSearchCollector(serviceId);
|
||||
}
|
||||
|
||||
protected InfoItemSearchCollector getInfoItemSearchCollector() {
|
||||
|
@ -49,7 +49,7 @@ public class SearchResult {
|
||||
return result;
|
||||
}
|
||||
|
||||
public String suggestion = "";
|
||||
public String suggestion;
|
||||
public List<InfoItem> resultList = new Vector<>();
|
||||
public List<Throwable> errors = new Vector<>();
|
||||
}
|
||||
|
160
services/youtube/ItagItem.java
Normal file
160
services/youtube/ItagItem.java
Normal file
@ -0,0 +1,160 @@
|
||||
package org.schabi.newpipe.extractor.services.youtube;
|
||||
|
||||
import org.schabi.newpipe.extractor.MediaFormat;
|
||||
import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
||||
|
||||
import static org.schabi.newpipe.extractor.MediaFormat.M4A;
|
||||
import static org.schabi.newpipe.extractor.MediaFormat.MPEG_4;
|
||||
import static org.schabi.newpipe.extractor.MediaFormat.WEBM;
|
||||
import static org.schabi.newpipe.extractor.MediaFormat.WEBMA;
|
||||
import static org.schabi.newpipe.extractor.MediaFormat.v3GPP;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.ItagItem.ItagType.AUDIO;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.ItagItem.ItagType.VIDEO;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.ItagItem.ItagType.VIDEO_ONLY;
|
||||
|
||||
public class ItagItem {
|
||||
/**
|
||||
* List can be found here https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/youtube.py#L360
|
||||
*/
|
||||
private static final ItagItem[] ITAG_LIST = {
|
||||
/////////////////////////////////////////////////////
|
||||
// VIDEO ID Type Format Resolution FPS ///
|
||||
///////////////////////////////////////////////////
|
||||
new ItagItem(17, VIDEO, v3GPP, "144p"),
|
||||
new ItagItem(36, VIDEO, v3GPP, "240p"),
|
||||
|
||||
new ItagItem(18, VIDEO, MPEG_4, "360p"),
|
||||
new ItagItem(34, VIDEO, MPEG_4, "360p"),
|
||||
new ItagItem(35, VIDEO, MPEG_4, "480p"),
|
||||
new ItagItem(59, VIDEO, MPEG_4, "480p"),
|
||||
new ItagItem(78, VIDEO, MPEG_4, "480p"),
|
||||
new ItagItem(22, VIDEO, MPEG_4, "720p"),
|
||||
new ItagItem(37, VIDEO, MPEG_4, "1080p"),
|
||||
new ItagItem(38, VIDEO, MPEG_4, "1080p"),
|
||||
|
||||
new ItagItem(43, VIDEO, WEBM, "360p"),
|
||||
new ItagItem(44, VIDEO, WEBM, "480p"),
|
||||
new ItagItem(45, VIDEO, WEBM, "720p"),
|
||||
new ItagItem(46, VIDEO, WEBM, "1080p"),
|
||||
|
||||
////////////////////////////////////////////////////////////////////
|
||||
// AUDIO ID ItagType Format Bitrate ///
|
||||
//////////////////////////////////////////////////////////////////
|
||||
// Disable Opus codec as it's not well supported in older devices
|
||||
// new ItagItem(249, AUDIO, WEBMA, 50),
|
||||
// new ItagItem(250, AUDIO, WEBMA, 70),
|
||||
// new ItagItem(251, AUDIO, WEBMA, 16),
|
||||
new ItagItem(171, AUDIO, WEBMA, 128),
|
||||
new ItagItem(172, AUDIO, WEBMA, 256),
|
||||
new ItagItem(139, AUDIO, M4A, 48),
|
||||
new ItagItem(140, AUDIO, M4A, 128),
|
||||
new ItagItem(141, AUDIO, M4A, 256),
|
||||
|
||||
/// VIDEO ONLY ////////////////////////////////////////////
|
||||
// ID Type Format Resolution FPS ///
|
||||
/////////////////////////////////////////////////////////
|
||||
// Don't add VideoOnly streams that have normal variants
|
||||
new ItagItem(160, VIDEO_ONLY, MPEG_4, "144p"),
|
||||
new ItagItem(133, VIDEO_ONLY, MPEG_4, "240p"),
|
||||
// new ItagItem(134, VIDEO_ONLY, MPEG_4, "360p"),
|
||||
new ItagItem(135, VIDEO_ONLY, MPEG_4, "480p"),
|
||||
new ItagItem(212, VIDEO_ONLY, MPEG_4, "480p"),
|
||||
// new ItagItem(136, VIDEO_ONLY, MPEG_4, "720p"),
|
||||
new ItagItem(298, VIDEO_ONLY, MPEG_4, "720p60", 60),
|
||||
new ItagItem(137, VIDEO_ONLY, MPEG_4, "1080p"),
|
||||
new ItagItem(299, VIDEO_ONLY, MPEG_4, "1080p60", 60),
|
||||
new ItagItem(266, VIDEO_ONLY, MPEG_4, "2160p"),
|
||||
|
||||
new ItagItem(278, VIDEO_ONLY, WEBM, "144p"),
|
||||
new ItagItem(242, VIDEO_ONLY, WEBM, "240p"),
|
||||
// new ItagItem(243, VIDEO_ONLY, WEBM, "360p"),
|
||||
new ItagItem(244, VIDEO_ONLY, WEBM, "480p"),
|
||||
new ItagItem(245, VIDEO_ONLY, WEBM, "480p"),
|
||||
new ItagItem(246, VIDEO_ONLY, WEBM, "480p"),
|
||||
new ItagItem(247, VIDEO_ONLY, WEBM, "720p"),
|
||||
new ItagItem(248, VIDEO_ONLY, WEBM, "1080p"),
|
||||
new ItagItem(271, VIDEO_ONLY, WEBM, "1440p"),
|
||||
// #272 is either 3840x2160 (e.g. RtoitU2A-3E) or 7680x4320 (sLprVF6d7Ug)
|
||||
new ItagItem(272, VIDEO_ONLY, WEBM, "2160p"),
|
||||
new ItagItem(302, VIDEO_ONLY, WEBM, "720p60", 60),
|
||||
new ItagItem(303, VIDEO_ONLY, WEBM, "1080p60", 60),
|
||||
new ItagItem(308, VIDEO_ONLY, WEBM, "1440p60", 60),
|
||||
new ItagItem(313, VIDEO_ONLY, WEBM, "2160p"),
|
||||
new ItagItem(315, VIDEO_ONLY, WEBM, "2160p60", 60)
|
||||
};
|
||||
|
||||
/*//////////////////////////////////////////////////////////////////////////
|
||||
// Utils
|
||||
//////////////////////////////////////////////////////////////////////////*/
|
||||
|
||||
public static boolean isSupported(int itag) {
|
||||
for (ItagItem item : ITAG_LIST) {
|
||||
if (itag == item.id) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
public static ItagItem getItag(int itagId) throws ParsingException {
|
||||
for (ItagItem item : ITAG_LIST) {
|
||||
if (itagId == item.id) {
|
||||
return item;
|
||||
}
|
||||
}
|
||||
throw new ParsingException("itag=" + Integer.toString(itagId) + " not supported");
|
||||
}
|
||||
|
||||
/*//////////////////////////////////////////////////////////////////////////
|
||||
// Contructors and misc
|
||||
//////////////////////////////////////////////////////////////////////////*/
|
||||
|
||||
public enum ItagType {
|
||||
AUDIO,
|
||||
VIDEO,
|
||||
VIDEO_ONLY
|
||||
}
|
||||
|
||||
/**
|
||||
* Call {@link #ItagItem(int, ItagType, MediaFormat, String, int)} with the fps set to 30.
|
||||
*/
|
||||
public ItagItem(int id, ItagType type, MediaFormat format, String resolution) {
|
||||
this.id = id;
|
||||
this.itagType = type;
|
||||
this.mediaFormatId = format.id;
|
||||
this.resolutionString = resolution;
|
||||
this.fps = 30;
|
||||
}
|
||||
|
||||
/**
|
||||
* Constructor for videos.
|
||||
*
|
||||
* @param resolution string that will be used in the frontend
|
||||
*/
|
||||
public ItagItem(int id, ItagType type, MediaFormat format, String resolution, int fps) {
|
||||
this.id = id;
|
||||
this.itagType = type;
|
||||
this.mediaFormatId = format.id;
|
||||
this.resolutionString = resolution;
|
||||
this.fps = fps;
|
||||
}
|
||||
|
||||
public ItagItem(int id, ItagType type, MediaFormat format, int avgBitrate) {
|
||||
this.id = id;
|
||||
this.itagType = type;
|
||||
this.mediaFormatId = format.id;
|
||||
this.avgBitrate = avgBitrate;
|
||||
}
|
||||
|
||||
public int id;
|
||||
public ItagType itagType;
|
||||
public int mediaFormatId;
|
||||
|
||||
// Audio fields
|
||||
public int avgBitrate = -1;
|
||||
|
||||
// Video fields
|
||||
public String resolutionString;
|
||||
public int fps = -1;
|
||||
|
||||
}
|
@ -13,10 +13,11 @@ import org.schabi.newpipe.extractor.channel.ChannelExtractor;
|
||||
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
|
||||
import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
||||
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
|
||||
import org.schabi.newpipe.extractor.stream.AbstractStreamInfo;
|
||||
import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector;
|
||||
import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor;
|
||||
import org.schabi.newpipe.extractor.stream.StreamType;
|
||||
import org.schabi.newpipe.extractor.utils.Parser;
|
||||
import org.schabi.newpipe.extractor.utils.Utils;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
@ -135,7 +136,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
|
||||
if (subscriberCount == -1) {
|
||||
Element el = doc.select("span[class*=\"yt-subscription-button-subscriber-count\"]").first();
|
||||
if (el != null) {
|
||||
subscriberCount = Long.parseLong(el.text().replaceAll("\\D+", ""));
|
||||
subscriberCount = Long.parseLong(Utils.removeNonDigitCharacters(el.text()));
|
||||
} else {
|
||||
throw new ParsingException("Could not get subscriber count");
|
||||
}
|
||||
@ -164,7 +165,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
|
||||
throw new ExtractionException("Channel doesn't have more streams");
|
||||
}
|
||||
|
||||
StreamInfoItemCollector collector = new StreamInfoItemCollector(getUrlIdHandler(), getServiceId());
|
||||
StreamInfoItemCollector collector = new StreamInfoItemCollector(getServiceId());
|
||||
setupNextStreamsAjax(NewPipe.getDownloader());
|
||||
collectStreamsFrom(collector, nextStreamsAjax.select("body").first());
|
||||
|
||||
@ -223,8 +224,8 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
|
||||
if (li.select("div[class=\"feed-item-dismissable\"]").first() != null) {
|
||||
collector.commit(new StreamInfoItemExtractor() {
|
||||
@Override
|
||||
public AbstractStreamInfo.StreamType getStreamType() throws ParsingException {
|
||||
return AbstractStreamInfo.StreamType.VIDEO_STREAM;
|
||||
public StreamType getStreamType() throws ParsingException {
|
||||
return StreamType.VIDEO_STREAM;
|
||||
}
|
||||
|
||||
@Override
|
||||
@ -302,7 +303,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
|
||||
return -1;
|
||||
}
|
||||
|
||||
output = input.replaceAll("\\D+", "");
|
||||
output = Utils.removeNonDigitCharacters(input);
|
||||
|
||||
try {
|
||||
return Long.parseLong(output);
|
||||
|
@ -3,6 +3,7 @@ package org.schabi.newpipe.extractor.services.youtube;
|
||||
import org.jsoup.nodes.Element;
|
||||
import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor;
|
||||
import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
||||
import org.schabi.newpipe.extractor.utils.Utils;
|
||||
|
||||
/*
|
||||
* Created by Christian Schabesberger on 12.02.17.
|
||||
@ -62,7 +63,7 @@ public class YoutubeChannelInfoItemExtractor implements ChannelInfoItemExtractor
|
||||
if (subsEl == null) {
|
||||
return 0;
|
||||
} else {
|
||||
return Long.parseLong(subsEl.text().replaceAll("\\D+", ""));
|
||||
return Long.parseLong(Utils.removeNonDigitCharacters(subsEl.text()));
|
||||
}
|
||||
}
|
||||
|
||||
@ -72,7 +73,7 @@ public class YoutubeChannelInfoItemExtractor implements ChannelInfoItemExtractor
|
||||
if (metaEl == null) {
|
||||
return 0;
|
||||
} else {
|
||||
return Long.parseLong(metaEl.text().replaceAll("\\D+", ""));
|
||||
return Long.parseLong(Utils.removeNonDigitCharacters(metaEl.text()));
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -26,18 +26,29 @@ import org.schabi.newpipe.extractor.utils.Parser;
|
||||
|
||||
public class YoutubeChannelUrlIdHandler implements UrlIdHandler {
|
||||
|
||||
private static final YoutubeChannelUrlIdHandler instance = new YoutubeChannelUrlIdHandler();
|
||||
private static final String ID_PATTERN = "/(user/[A-Za-z0-9_-]*|channel/[A-Za-z0-9_-]*)";
|
||||
|
||||
public static YoutubeChannelUrlIdHandler getInstance() {
|
||||
return instance;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getUrl(String channelId) {
|
||||
return "https://www.youtube.com/" + channelId;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getId(String siteUrl) throws ParsingException {
|
||||
return Parser.matchGroup1("/(user/[A-Za-z0-9_-]*|channel/[A-Za-z0-9_-]*)", siteUrl);
|
||||
return Parser.matchGroup1(ID_PATTERN, siteUrl);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String cleanUrl(String siteUrl) throws ParsingException {
|
||||
return getUrl(getId(siteUrl));
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean acceptUrl(String videoUrl) {
|
||||
return (videoUrl.contains("youtube") ||
|
||||
videoUrl.contains("youtu.be")) &&
|
||||
|
@ -12,10 +12,11 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException;
|
||||
import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
||||
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
|
||||
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
|
||||
import org.schabi.newpipe.extractor.stream.AbstractStreamInfo;
|
||||
import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector;
|
||||
import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor;
|
||||
import org.schabi.newpipe.extractor.stream.StreamType;
|
||||
import org.schabi.newpipe.extractor.utils.Parser;
|
||||
import org.schabi.newpipe.extractor.utils.Utils;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
@ -157,7 +158,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
|
||||
}
|
||||
|
||||
try {
|
||||
streamsCount = Long.parseLong(input.replaceAll("\\D+", ""));
|
||||
streamsCount = Long.parseLong(Utils.removeNonDigitCharacters(input));
|
||||
} catch (NumberFormatException e) {
|
||||
// When there's no videos in a playlist, there's no number in the "innerHtml",
|
||||
// all characters that is not a number is removed, so we try to parse a empty string
|
||||
@ -186,7 +187,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
|
||||
throw new ExtractionException("Playlist doesn't have more streams");
|
||||
}
|
||||
|
||||
StreamInfoItemCollector collector = new StreamInfoItemCollector(getUrlIdHandler(), getServiceId());
|
||||
StreamInfoItemCollector collector = new StreamInfoItemCollector(getServiceId());
|
||||
setupNextStreamsAjax(NewPipe.getDownloader());
|
||||
collectStreamsFrom(collector, nextStreamsAjax.select("tbody[id=\"pl-load-more-destination\"]").first());
|
||||
|
||||
@ -244,8 +245,8 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
|
||||
for (final Element li : element.children()) {
|
||||
collector.commit(new StreamInfoItemExtractor() {
|
||||
@Override
|
||||
public AbstractStreamInfo.StreamType getStreamType() throws ParsingException {
|
||||
return AbstractStreamInfo.StreamType.VIDEO_STREAM;
|
||||
public StreamType getStreamType() throws ParsingException {
|
||||
return StreamType.VIDEO_STREAM;
|
||||
}
|
||||
|
||||
@Override
|
||||
|
@ -7,8 +7,13 @@ import org.schabi.newpipe.extractor.utils.Parser;
|
||||
|
||||
public class YoutubePlaylistUrlIdHandler implements UrlIdHandler {
|
||||
|
||||
private static final YoutubePlaylistUrlIdHandler instance = new YoutubePlaylistUrlIdHandler();
|
||||
private static final String ID_PATTERN = "([\\-a-zA-Z0-9_]{34})";
|
||||
|
||||
public static YoutubePlaylistUrlIdHandler getInstance() {
|
||||
return instance;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getUrl(String listId) {
|
||||
return "https://www.youtube.com/playlist?list=" + listId;
|
||||
|
@ -5,7 +5,6 @@ import org.jsoup.nodes.Document;
|
||||
import org.jsoup.nodes.Element;
|
||||
import org.schabi.newpipe.extractor.Downloader;
|
||||
import org.schabi.newpipe.extractor.NewPipe;
|
||||
import org.schabi.newpipe.extractor.UrlIdHandler;
|
||||
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
|
||||
import org.schabi.newpipe.extractor.search.InfoItemSearchCollector;
|
||||
import org.schabi.newpipe.extractor.search.SearchEngine;
|
||||
@ -40,8 +39,8 @@ public class YoutubeSearchEngine extends SearchEngine {
|
||||
private static final String TAG = YoutubeSearchEngine.class.toString();
|
||||
public static final String CHARSET_UTF_8 = "UTF-8";
|
||||
|
||||
public YoutubeSearchEngine(UrlIdHandler urlIdHandler, int serviceId) {
|
||||
super(urlIdHandler, serviceId);
|
||||
public YoutubeSearchEngine(int serviceId) {
|
||||
super(serviceId);
|
||||
}
|
||||
|
||||
@Override
|
||||
|
@ -58,7 +58,7 @@ public class YoutubeService extends StreamingService {
|
||||
|
||||
@Override
|
||||
public SearchEngine getSearchEngineInstance() {
|
||||
return new YoutubeSearchEngine(getStreamUrlIdHandlerInstance(), getServiceId());
|
||||
return new YoutubeSearchEngine(getServiceId());
|
||||
}
|
||||
|
||||
@Override
|
||||
@ -68,13 +68,13 @@ public class YoutubeService extends StreamingService {
|
||||
|
||||
@Override
|
||||
public UrlIdHandler getChannelUrlIdHandlerInstance() {
|
||||
return new YoutubeChannelUrlIdHandler();
|
||||
return YoutubeChannelUrlIdHandler.getInstance();
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public UrlIdHandler getPlaylistUrlIdHandlerInstance() {
|
||||
return new YoutubePlaylistUrlIdHandler();
|
||||
return YoutubePlaylistUrlIdHandler.getInstance();
|
||||
}
|
||||
|
||||
@Override
|
||||
|
@ -9,20 +9,21 @@ import org.mozilla.javascript.Context;
|
||||
import org.mozilla.javascript.Function;
|
||||
import org.mozilla.javascript.ScriptableObject;
|
||||
import org.schabi.newpipe.extractor.Downloader;
|
||||
import org.schabi.newpipe.extractor.MediaFormat;
|
||||
import org.schabi.newpipe.extractor.NewPipe;
|
||||
import org.schabi.newpipe.extractor.UrlIdHandler;
|
||||
import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
|
||||
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
|
||||
import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
||||
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
|
||||
import org.schabi.newpipe.extractor.stream.AbstractStreamInfo;
|
||||
import org.schabi.newpipe.extractor.stream.AudioStream;
|
||||
import org.schabi.newpipe.extractor.stream.Stream;
|
||||
import org.schabi.newpipe.extractor.stream.StreamExtractor;
|
||||
import org.schabi.newpipe.extractor.stream.StreamInfo;
|
||||
import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector;
|
||||
import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor;
|
||||
import org.schabi.newpipe.extractor.stream.StreamType;
|
||||
import org.schabi.newpipe.extractor.stream.VideoStream;
|
||||
import org.schabi.newpipe.extractor.utils.Parser;
|
||||
import org.schabi.newpipe.extractor.utils.Utils;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.List;
|
||||
@ -52,12 +53,11 @@ import java.util.regex.Pattern;
|
||||
*/
|
||||
|
||||
public class YoutubeStreamExtractor extends StreamExtractor {
|
||||
public static final String URL_ENCODED_FMT_STREAM_MAP = "url_encoded_fmt_stream_map";
|
||||
public static final String HTTPS = "https:";
|
||||
public static final String CONTENT = "content";
|
||||
public static final String REGEX_INT = "[^\\d]";
|
||||
private static final String TAG = YoutubeStreamExtractor.class.getSimpleName();
|
||||
|
||||
// exceptions
|
||||
/*//////////////////////////////////////////////////////////////////////////
|
||||
// Exceptions
|
||||
//////////////////////////////////////////////////////////////////////////*/
|
||||
|
||||
public class DecryptException extends ParsingException {
|
||||
DecryptException(String message, Throwable cause) {
|
||||
@ -65,8 +65,6 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
||||
}
|
||||
}
|
||||
|
||||
// special content not available exceptions
|
||||
|
||||
public class GemaException extends ContentNotAvailableException {
|
||||
GemaException(String message) {
|
||||
super(message);
|
||||
@ -79,267 +77,27 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
||||
}
|
||||
}
|
||||
|
||||
// ----------------
|
||||
/*//////////////////////////////////////////////////////////////////////////*/
|
||||
|
||||
// Sometimes if the html page of youtube is already downloaded, youtube web page will internally
|
||||
// download the /get_video_info page. Since a certain date dashmpd url is only available over
|
||||
// this /get_video_info page, so we always need to download this one to.
|
||||
// %%video_id%% will be replaced by the actual video id
|
||||
// $$el_type$$ will be replaced by the actual el_type (se the declarations below)
|
||||
private static final String GET_VIDEO_INFO_URL =
|
||||
"https://www.youtube.com/get_video_info?video_id=%%video_id%%$$el_type$$&ps=default&eurl=&gl=US&hl=en";
|
||||
// eltype is necessary for the url above
|
||||
private static final String EL_INFO = "el=info";
|
||||
private Document doc;
|
||||
private final String dirtyUrl;
|
||||
|
||||
public enum ItagType {
|
||||
AUDIO,
|
||||
VIDEO,
|
||||
VIDEO_ONLY
|
||||
public YoutubeStreamExtractor(UrlIdHandler urlIdHandler, String pageUrl, int serviceId) throws ExtractionException, IOException {
|
||||
super(urlIdHandler, urlIdHandler.cleanUrl(pageUrl), serviceId);
|
||||
dirtyUrl = pageUrl;
|
||||
fetchDocument();
|
||||
}
|
||||
|
||||
private static class ItagItem {
|
||||
public ItagItem(int id, ItagType type, MediaFormat format, String res, int fps) {
|
||||
this.id = id;
|
||||
this.itagType = type;
|
||||
this.mediaFormatId = format.id;
|
||||
this.resolutionString = res;
|
||||
this.fps = fps;
|
||||
}
|
||||
/*//////////////////////////////////////////////////////////////////////////
|
||||
// Impl
|
||||
//////////////////////////////////////////////////////////////////////////*/
|
||||
|
||||
public ItagItem(int id, ItagType type, MediaFormat format, int samplingRate, int bandWidth) {
|
||||
this(id, type, format, 0, samplingRate, bandWidth);
|
||||
}
|
||||
|
||||
public ItagItem(int id, ItagType type, MediaFormat format, int avgBitrate, int samplingRate, int bandWidth) {
|
||||
this.id = id;
|
||||
this.itagType = type;
|
||||
this.mediaFormatId = format.id;
|
||||
this.avgBitrate = avgBitrate;
|
||||
this.samplingRate = samplingRate;
|
||||
this.bandWidth = bandWidth;
|
||||
}
|
||||
|
||||
public int id;
|
||||
public ItagType itagType;
|
||||
public int mediaFormatId;
|
||||
public String resolutionString;
|
||||
public int fps = -1;
|
||||
public int avgBitrate = -1;
|
||||
public int samplingRate = -1;
|
||||
public int bandWidth = -1;
|
||||
}
|
||||
|
||||
/**
|
||||
* List can be found here https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/youtube.py#L360
|
||||
*/
|
||||
private static final ItagItem[] itagList = {
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
// VIDEO ID ItagType Format Resolution FPS ///
|
||||
////////////////////////////////////////////////////////////////////////
|
||||
new ItagItem(17, ItagType.VIDEO, MediaFormat.v3GPP, "144p" , 12),
|
||||
new ItagItem(18, ItagType.VIDEO, MediaFormat.MPEG_4, "360p" , 24),
|
||||
new ItagItem(22, ItagType.VIDEO, MediaFormat.MPEG_4, "720p" , 24),
|
||||
new ItagItem(36, ItagType.VIDEO, MediaFormat.v3GPP, "240p" , 24),
|
||||
new ItagItem(37, ItagType.VIDEO, MediaFormat.MPEG_4, "1080p" , 24),
|
||||
new ItagItem(38, ItagType.VIDEO, MediaFormat.MPEG_4, "1080p" , 24),
|
||||
new ItagItem(43, ItagType.VIDEO, MediaFormat.WEBM, "360p" , 24),
|
||||
new ItagItem(44, ItagType.VIDEO, MediaFormat.WEBM, "480p" , 24),
|
||||
new ItagItem(45, ItagType.VIDEO, MediaFormat.WEBM, "720p" , 24),
|
||||
new ItagItem(46, ItagType.VIDEO, MediaFormat.WEBM, "1080p" , 24),
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////////////////////
|
||||
// AUDIO ID ItagType Format Bitrate SamplingR Bandwidth ///
|
||||
////////////////////////////////////////////////////////////////////////////////////////
|
||||
// Disable Opus codec as it's not well supported in older devices
|
||||
// new ItagItem(249, ItagType.AUDIO, MediaFormat.WEBMA, 50, 0, 0),
|
||||
// new ItagItem(250, ItagType.AUDIO, MediaFormat.WEBMA, 70, 0, 0),
|
||||
// new ItagItem(251, ItagType.AUDIO, MediaFormat.WEBMA, 160, 0, 0),
|
||||
new ItagItem(171, ItagType.AUDIO, MediaFormat.WEBMA, 128, 0, 0),
|
||||
new ItagItem(172, ItagType.AUDIO, MediaFormat.WEBMA, 256, 0, 0),
|
||||
new ItagItem(140, ItagType.AUDIO, MediaFormat.M4A, 128, 0, 0),
|
||||
new ItagItem(141, ItagType.AUDIO, MediaFormat.M4A, 256, 0, 0),
|
||||
|
||||
/// VIDEO ONLY ///////////////////////////////////////////////////////////////////
|
||||
// ID ItagType Format Resolution FPS ///
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Don't add VideoOnly streams that have normal variants
|
||||
// new ItagItem(160, ItagType.VIDEO_ONLY, MediaFormat.MPEG_4, "144p" , 24),
|
||||
// new ItagItem(133, ItagType.VIDEO_ONLY, MediaFormat.MPEG_4, "240p" , 24),
|
||||
// new ItagItem(134, ItagType.VIDEO_ONLY, MediaFormat.MPEG_4, "360p" , 24),
|
||||
new ItagItem(135, ItagType.VIDEO_ONLY, MediaFormat.MPEG_4, "480p" , 30),
|
||||
// new ItagItem(136, ItagType.VIDEO_ONLY, MediaFormat.MPEG_4, "720p" , 30),
|
||||
new ItagItem(298, ItagType.VIDEO_ONLY, MediaFormat.MPEG_4, "720p60" , 60),
|
||||
new ItagItem(137, ItagType.VIDEO_ONLY, MediaFormat.MPEG_4, "1080p" , 30),
|
||||
new ItagItem(299, ItagType.VIDEO_ONLY, MediaFormat.MPEG_4, "1080p60" , 60),
|
||||
new ItagItem(266, ItagType.VIDEO_ONLY, MediaFormat.MPEG_4, "2160p" , 30),
|
||||
|
||||
// new ItagItem(243, ItagType.VIDEO_ONLY, MediaFormat.WEBM, "360p" , 30),
|
||||
new ItagItem(244, ItagType.VIDEO_ONLY, MediaFormat.WEBM, "480p" , 30),
|
||||
new ItagItem(245, ItagType.VIDEO_ONLY, MediaFormat.WEBM, "480p" , 30),
|
||||
new ItagItem(246, ItagType.VIDEO_ONLY, MediaFormat.WEBM, "480p" , 30),
|
||||
new ItagItem(247, ItagType.VIDEO_ONLY, MediaFormat.WEBM, "720p" , 30),
|
||||
new ItagItem(248, ItagType.VIDEO_ONLY, MediaFormat.WEBM, "1080p" , 30),
|
||||
new ItagItem(271, ItagType.VIDEO_ONLY, MediaFormat.WEBM, "1440p" , 30),
|
||||
// #272 is either 3840x2160 (e.g. RtoitU2A-3E) or 7680x4320 (sLprVF6d7Ug)
|
||||
new ItagItem(272, ItagType.VIDEO_ONLY, MediaFormat.WEBM, "2160p" , 30),
|
||||
new ItagItem(302, ItagType.VIDEO_ONLY, MediaFormat.WEBM, "720p60" , 60),
|
||||
new ItagItem(303, ItagType.VIDEO_ONLY, MediaFormat.WEBM, "1080p60" , 60),
|
||||
new ItagItem(308, ItagType.VIDEO_ONLY, MediaFormat.WEBM, "1440p60" , 60),
|
||||
new ItagItem(313, ItagType.VIDEO_ONLY, MediaFormat.WEBM, "2160p" , 30),
|
||||
new ItagItem(315, ItagType.VIDEO_ONLY, MediaFormat.WEBM, "2160p60" , 60)
|
||||
};
|
||||
|
||||
public static boolean itagIsSupported(int itag) {
|
||||
for (ItagItem item : itagList) {
|
||||
if (itag == item.id) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
public static ItagItem getItagItem(int itag) throws ParsingException {
|
||||
for (ItagItem item : itagList) {
|
||||
if (itag == item.id) {
|
||||
return item;
|
||||
}
|
||||
}
|
||||
throw new ParsingException("itag=" + Integer.toString(itag) + " not supported");
|
||||
}
|
||||
|
||||
private static final String TAG = YoutubeStreamExtractor.class.toString();
|
||||
private final Document doc;
|
||||
private JSONObject playerArgs;
|
||||
private boolean isAgeRestricted;
|
||||
private Map<String, String> videoInfoPage;
|
||||
|
||||
// static values
|
||||
private static final String DECRYPTION_FUNC_NAME = "decrypt";
|
||||
|
||||
// cached values
|
||||
private static volatile String decryptionCode = "";
|
||||
|
||||
UrlIdHandler urlidhandler = YoutubeStreamUrlIdHandler.getInstance();
|
||||
String pageUrl = "";
|
||||
|
||||
public YoutubeStreamExtractor(UrlIdHandler urlIdHandler, String pageUrl, int serviceId)
|
||||
throws ExtractionException, IOException {
|
||||
super(urlIdHandler, pageUrl, serviceId);
|
||||
//most common videoInfo fields are now set in our superclass, for all services
|
||||
this.pageUrl = pageUrl;
|
||||
Downloader downloader = NewPipe.getDownloader();
|
||||
String pageContent = downloader.download(urlidhandler.cleanUrl(pageUrl));
|
||||
doc = Jsoup.parse(pageContent, pageUrl);
|
||||
JSONObject ytPlayerConfig;
|
||||
String playerUrl;
|
||||
|
||||
// Check if the video is age restricted
|
||||
if (pageContent.contains("<meta property=\"og:restrictions:age")) {
|
||||
String videoInfoUrl = GET_VIDEO_INFO_URL.replace("%%video_id%%",
|
||||
urlidhandler.getId(pageUrl)).replace("$$el_type$$", "&" + EL_INFO);
|
||||
String videoInfoPageString = downloader.download(videoInfoUrl);
|
||||
videoInfoPage = Parser.compatParseMap(videoInfoPageString);
|
||||
playerUrl = getPlayerUrlFromRestrictedVideo(pageUrl);
|
||||
isAgeRestricted = true;
|
||||
} else {
|
||||
ytPlayerConfig = getPlayerConfig(pageContent);
|
||||
playerArgs = getPlayerArgs(ytPlayerConfig);
|
||||
playerUrl = getPlayerUrl(ytPlayerConfig);
|
||||
isAgeRestricted = false;
|
||||
}
|
||||
|
||||
if (decryptionCode.isEmpty()) {
|
||||
decryptionCode = loadDecryptionCode(playerUrl);
|
||||
}
|
||||
}
|
||||
|
||||
private JSONObject getPlayerConfig(String pageContent) throws ParsingException {
|
||||
@Override
|
||||
public String getId() throws ParsingException {
|
||||
try {
|
||||
String ytPlayerConfigRaw =
|
||||
Parser.matchGroup1("ytplayer.config\\s*=\\s*(\\{.*?\\});", pageContent);
|
||||
return new JSONObject(ytPlayerConfigRaw);
|
||||
} catch (Parser.RegexException e) {
|
||||
String errorReason = getErrorMessage();
|
||||
switch(errorReason) {
|
||||
case "GEMA":
|
||||
throw new GemaException(errorReason);
|
||||
case "":
|
||||
throw new ContentNotAvailableException("Content not available: player config empty", e);
|
||||
default:
|
||||
throw new ContentNotAvailableException("Content not available", e);
|
||||
}
|
||||
} catch (JSONException e) {
|
||||
throw new ParsingException("Could not parse yt player config", e);
|
||||
}
|
||||
}
|
||||
|
||||
private JSONObject getPlayerArgs(JSONObject playerConfig) throws ParsingException {
|
||||
JSONObject playerArgs;
|
||||
|
||||
//attempt to load the youtube js player JSON arguments
|
||||
boolean isLiveStream = false; //used to determine if this is a livestream or not
|
||||
try {
|
||||
playerArgs = playerConfig.getJSONObject("args");
|
||||
|
||||
// check if we have a live stream. We need to filter it, since its not yet supported.
|
||||
if ((playerArgs.has("ps") && playerArgs.get("ps").toString().equals("live"))
|
||||
|| (playerArgs.get(URL_ENCODED_FMT_STREAM_MAP).toString().isEmpty())) {
|
||||
isLiveStream = true;
|
||||
}
|
||||
} catch (JSONException e) {
|
||||
throw new ParsingException("Could not parse yt player config", e);
|
||||
}
|
||||
if (isLiveStream) {
|
||||
throw new LiveStreamException("This is a Life stream. Can't use those right now.");
|
||||
}
|
||||
|
||||
return playerArgs;
|
||||
}
|
||||
|
||||
private String getPlayerUrl(JSONObject playerConfig) throws ParsingException {
|
||||
try {
|
||||
// The Youtube service needs to be initialized by downloading the
|
||||
// js-Youtube-player. This is done in order to get the algorithm
|
||||
// for decrypting cryptic signatures inside certain stream urls.
|
||||
String playerUrl = "";
|
||||
|
||||
JSONObject ytAssets = playerConfig.getJSONObject("assets");
|
||||
playerUrl = ytAssets.getString("js");
|
||||
|
||||
if (playerUrl.startsWith("//")) {
|
||||
playerUrl = HTTPS + playerUrl;
|
||||
}
|
||||
return playerUrl;
|
||||
} catch (JSONException e) {
|
||||
throw new ParsingException(
|
||||
"Could not load decryption code for the Youtube service.", e);
|
||||
}
|
||||
}
|
||||
|
||||
private String getPlayerUrlFromRestrictedVideo(String pageUrl) throws ParsingException, ReCaptchaException {
|
||||
try {
|
||||
Downloader downloader = NewPipe.getDownloader();
|
||||
String playerUrl = "";
|
||||
String videoId = urlidhandler.getId(pageUrl);
|
||||
String embedUrl = "https://www.youtube.com/embed/" + videoId;
|
||||
String embedPageContent = downloader.download(embedUrl);
|
||||
//todo: find out if this can be reapaced by Parser.matchGroup1()
|
||||
Pattern assetsPattern = Pattern.compile("\"assets\":.+?\"js\":\\s*(\"[^\"]+\")");
|
||||
Matcher patternMatcher = assetsPattern.matcher(embedPageContent);
|
||||
while (patternMatcher.find()) {
|
||||
playerUrl = patternMatcher.group(1);
|
||||
}
|
||||
playerUrl = playerUrl.replace("\\", "").replace("\"", "");
|
||||
|
||||
if (playerUrl.startsWith("//")) {
|
||||
playerUrl = HTTPS + playerUrl;
|
||||
}
|
||||
return playerUrl;
|
||||
} catch (IOException e) {
|
||||
throw new ParsingException(
|
||||
"Could load decryption code form restricted video for the Youtube service.", e);
|
||||
} catch (ReCaptchaException e) {
|
||||
throw new ReCaptchaException("reCaptcha Challenge requested");
|
||||
return getUrlIdHandler().getId(getUrl());
|
||||
} catch (Exception e) {
|
||||
throw new ParsingException("Could not get stream id");
|
||||
}
|
||||
}
|
||||
|
||||
@ -457,7 +215,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
||||
@Override
|
||||
public String getDashMpdUrl() throws ParsingException {
|
||||
try {
|
||||
String dashManifestUrl = "";
|
||||
String dashManifestUrl;
|
||||
if (videoInfoPage != null && videoInfoPage.containsKey("dashmpd")) {
|
||||
dashManifestUrl = videoInfoPage.get("dashmpd");
|
||||
} else if (playerArgs.has("dashmpd")) {
|
||||
@ -479,7 +237,6 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public List<AudioStream> getAudioStreams() throws ParsingException {
|
||||
Vector<AudioStream> audioStreams = new Vector<>();
|
||||
@ -507,9 +264,9 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
||||
|
||||
int itag = Integer.parseInt(tags.get("itag"));
|
||||
|
||||
if (itagIsSupported(itag)) {
|
||||
ItagItem itagItem = getItagItem(itag);
|
||||
if (itagItem.itagType == ItagType.AUDIO) {
|
||||
if (ItagItem.isSupported(itag)) {
|
||||
ItagItem itagItem = ItagItem.getItag(itag);
|
||||
if (itagItem.itagType == ItagItem.ItagType.AUDIO) {
|
||||
String streamUrl = tags.get("url");
|
||||
// if video has a signature: decrypt it and add it to the url
|
||||
if (tags.get("s") != null) {
|
||||
@ -517,11 +274,10 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
||||
+ decryptSignature(tags.get("s"), decryptionCode);
|
||||
}
|
||||
|
||||
audioStreams.add(new AudioStream(streamUrl,
|
||||
itagItem.mediaFormatId,
|
||||
itagItem.avgBitrate,
|
||||
itagItem.bandWidth,
|
||||
itagItem.samplingRate));
|
||||
AudioStream audioStream = new AudioStream(streamUrl, itagItem.mediaFormatId, itagItem.avgBitrate);
|
||||
if (!Stream.containSimilarStream(audioStream, audioStreams)) {
|
||||
audioStreams.add(audioStream);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -552,19 +308,20 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
||||
|
||||
int itag = Integer.parseInt(tags.get("itag"));
|
||||
|
||||
if (itagIsSupported(itag)) {
|
||||
ItagItem itagItem = getItagItem(itag);
|
||||
if (itagItem.itagType == ItagType.VIDEO) {
|
||||
if (ItagItem.isSupported(itag)) {
|
||||
ItagItem itagItem = ItagItem.getItag(itag);
|
||||
if (itagItem.itagType == ItagItem.ItagType.VIDEO) {
|
||||
String streamUrl = tags.get("url");
|
||||
// if video has a signature: decrypt it and add it to the url
|
||||
if (tags.get("s") != null) {
|
||||
streamUrl = streamUrl + "&signature="
|
||||
+ decryptSignature(tags.get("s"), decryptionCode);
|
||||
}
|
||||
videoStreams.add(new VideoStream(
|
||||
streamUrl,
|
||||
itagItem.mediaFormatId,
|
||||
itagItem.resolutionString));
|
||||
|
||||
VideoStream videoStream = new VideoStream(streamUrl, itagItem.mediaFormatId, itagItem.resolutionString);
|
||||
if (!Stream.containSimilarStream(videoStream, videoStreams)) {
|
||||
videoStreams.add(videoStream);
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
@ -612,9 +369,9 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
||||
|
||||
int itag = Integer.parseInt(tags.get("itag"));
|
||||
|
||||
if (itagIsSupported(itag)) {
|
||||
ItagItem itagItem = getItagItem(itag);
|
||||
if (itagItem.itagType == ItagType.VIDEO_ONLY) {
|
||||
if (ItagItem.isSupported(itag)) {
|
||||
ItagItem itagItem = ItagItem.getItag(itag);
|
||||
if (itagItem.itagType == ItagItem.ItagType.VIDEO_ONLY) {
|
||||
String streamUrl = tags.get("url");
|
||||
// if video has a signature: decrypt it and add it to the url
|
||||
if (tags.get("s") != null) {
|
||||
@ -622,11 +379,10 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
||||
+ decryptSignature(tags.get("s"), decryptionCode);
|
||||
}
|
||||
|
||||
videoOnlyStreams.add(new VideoStream(
|
||||
true, //isVideoOnly
|
||||
streamUrl,
|
||||
itagItem.mediaFormatId,
|
||||
itagItem.resolutionString));
|
||||
VideoStream videoStream = new VideoStream(streamUrl, itagItem.mediaFormatId, itagItem.resolutionString, true);
|
||||
if (!Stream.containSimilarStream(videoStream, videoOnlyStreams)) {
|
||||
videoOnlyStreams.add(videoStream);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -649,7 +405,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
||||
public int getTimeStamp() throws ParsingException {
|
||||
String timeStamp;
|
||||
try {
|
||||
timeStamp = Parser.matchGroup1("((#|&|\\?)t=\\d{0,3}h?\\d{0,3}m?\\d{1,3}s?)", pageUrl);
|
||||
timeStamp = Parser.matchGroup1("((#|&|\\?)t=\\d{0,3}h?\\d{0,3}m?\\d{1,3}s?)", dirtyUrl);
|
||||
} catch (Parser.RegexException e) {
|
||||
// catch this instantly since an url does not necessarily have to have a time stamp
|
||||
|
||||
@ -730,7 +486,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
||||
//if this ckicks in our button has no content and thefore likes/dislikes are disabled
|
||||
return -1;
|
||||
}
|
||||
return Integer.parseInt(likesString.replaceAll(REGEX_INT, ""));
|
||||
return Integer.parseInt(Utils.removeNonDigitCharacters(likesString));
|
||||
} catch (NumberFormatException nfe) {
|
||||
throw new ParsingException(
|
||||
"failed to parse likesString \"" + likesString + "\" as integers", nfe);
|
||||
@ -750,7 +506,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
||||
//if this kicks in our button has no content and therefore likes/dislikes are disabled
|
||||
return -1;
|
||||
}
|
||||
return Integer.parseInt(dislikesString.replaceAll(REGEX_INT, ""));
|
||||
return Integer.parseInt(Utils.removeNonDigitCharacters(dislikesString));
|
||||
} catch (NumberFormatException nfe) {
|
||||
throw new ParsingException(
|
||||
"failed to parse dislikesString \"" + dislikesString + "\" as integers", nfe);
|
||||
@ -788,11 +544,6 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getPageUrl() {
|
||||
return pageUrl;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getChannelUrl() throws ParsingException {
|
||||
try {
|
||||
@ -804,92 +555,186 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
||||
}
|
||||
|
||||
@Override
|
||||
public StreamInfo.StreamType getStreamType() throws ParsingException {
|
||||
public StreamType getStreamType() throws ParsingException {
|
||||
//todo: if implementing livestream support this value should be generated dynamically
|
||||
return StreamInfo.StreamType.VIDEO_STREAM;
|
||||
return StreamType.VIDEO_STREAM;
|
||||
}
|
||||
|
||||
/**
|
||||
* Provides information about links to other videos on the video page, such as related videos.
|
||||
* This is encapsulated in a StreamInfoItem object,
|
||||
* which is a subset of the fields in a full StreamInfo.
|
||||
* {@inheritDoc}
|
||||
*/
|
||||
private StreamInfoItemExtractor extractVideoPreviewInfo(final Element li) {
|
||||
return new StreamInfoItemExtractor() {
|
||||
@Override
|
||||
public AbstractStreamInfo.StreamType getStreamType() throws ParsingException {
|
||||
return AbstractStreamInfo.StreamType.VIDEO_STREAM;
|
||||
}
|
||||
@Override
|
||||
public String getErrorMessage() {
|
||||
String errorMessage = doc.select("h1[id=\"unavailable-message\"]").first().text();
|
||||
StringBuilder errorReason;
|
||||
|
||||
@Override
|
||||
public boolean isAd() throws ParsingException {
|
||||
return !li.select("span[class*=\"icon-not-available\"]").isEmpty();
|
||||
}
|
||||
if (errorMessage == null || errorMessage.isEmpty()) {
|
||||
errorReason = null;
|
||||
} else if (errorMessage.contains("GEMA")) {
|
||||
// Gema sometimes blocks youtube music content in germany:
|
||||
// https://www.gema.de/en/
|
||||
// Detailed description:
|
||||
// https://en.wikipedia.org/wiki/GEMA_%28German_organization%29
|
||||
errorReason = new StringBuilder("GEMA");
|
||||
} else {
|
||||
errorReason = new StringBuilder(errorMessage);
|
||||
errorReason.append(" ");
|
||||
errorReason.append(doc.select("[id=\"unavailable-submessage\"]").first().text());
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getWebPageUrl() throws ParsingException {
|
||||
return li.select("a.content-link").first().attr("abs:href");
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getTitle() throws ParsingException {
|
||||
//todo: check NullPointerException causing
|
||||
return li.select("span.title").first().text();
|
||||
//this page causes the NullPointerException, after finding it by searching for "tjvg":
|
||||
//https://www.youtube.com/watch?v=Uqg0aEhLFAg
|
||||
}
|
||||
|
||||
@Override
|
||||
public int getDuration() throws ParsingException {
|
||||
return YoutubeParsingHelper.parseDurationString(
|
||||
li.select("span.video-time").first().text());
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getUploader() throws ParsingException {
|
||||
return li.select("span.g-hovercard").first().text();
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getUploadDate() throws ParsingException {
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getViewCount() throws ParsingException {
|
||||
//this line is unused
|
||||
//String views = li.select("span.view-count").first().text();
|
||||
|
||||
//Log.i(TAG, "title:"+info.title);
|
||||
//Log.i(TAG, "view count:"+views);
|
||||
|
||||
try {
|
||||
return Long.parseLong(li.select("span.view-count")
|
||||
.first().text().replaceAll(REGEX_INT, ""));
|
||||
} catch (Exception e) {
|
||||
//related videos sometimes have no view count
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getThumbnailUrl() throws ParsingException {
|
||||
Element img = li.select("img").first();
|
||||
String thumbnailUrl = img.attr("abs:src");
|
||||
// Sometimes youtube sends links to gif files which somehow seem to not exist
|
||||
// anymore. Items with such gif also offer a secondary image source. So we are going
|
||||
// to use that if we caught such an item.
|
||||
if (thumbnailUrl.contains(".gif")) {
|
||||
thumbnailUrl = img.attr("data-thumb");
|
||||
}
|
||||
if (thumbnailUrl.startsWith("//")) {
|
||||
thumbnailUrl = HTTPS + thumbnailUrl;
|
||||
}
|
||||
return thumbnailUrl;
|
||||
}
|
||||
};
|
||||
return errorReason != null ? errorReason.toString() : null;
|
||||
}
|
||||
|
||||
/*//////////////////////////////////////////////////////////////////////////
|
||||
// Utils
|
||||
//////////////////////////////////////////////////////////////////////////*/
|
||||
|
||||
private JSONObject playerArgs;
|
||||
private boolean isAgeRestricted;
|
||||
private Map<String, String> videoInfoPage;
|
||||
|
||||
private static final String URL_ENCODED_FMT_STREAM_MAP = "url_encoded_fmt_stream_map";
|
||||
private static final String HTTPS = "https:";
|
||||
private static final String CONTENT = "content";
|
||||
|
||||
/**
|
||||
* Sometimes if the html page of youtube is already downloaded, youtube web page will internally
|
||||
* download the /get_video_info page. Since a certain date dashmpd url is only available over
|
||||
* this /get_video_info page, so we always need to download this one to.
|
||||
* <p>
|
||||
* %%video_id%% will be replaced by the actual video id
|
||||
* $$el_type$$ will be replaced by the actual el_type (se the declarations below)
|
||||
*/
|
||||
private static final String GET_VIDEO_INFO_URL =
|
||||
"https://www.youtube.com/get_video_info?video_id=%%video_id%%$$el_type$$&ps=default&eurl=&gl=US&hl=en";
|
||||
// eltype is necessary for the url above
|
||||
private static final String EL_INFO = "el=info";
|
||||
|
||||
|
||||
// static values
|
||||
private static final String DECRYPTION_FUNC_NAME = "decrypt";
|
||||
|
||||
// cached values
|
||||
private static volatile String decryptionCode = "";
|
||||
|
||||
private void fetchDocument() throws IOException, ReCaptchaException, ParsingException {
|
||||
Downloader downloader = NewPipe.getDownloader();
|
||||
|
||||
String pageContent = downloader.download(getUrl());
|
||||
doc = Jsoup.parse(pageContent, getUrl());
|
||||
|
||||
JSONObject ytPlayerConfig;
|
||||
String playerUrl;
|
||||
|
||||
String videoInfoUrl = GET_VIDEO_INFO_URL.replace("%%video_id%%", getId()).replace("$$el_type$$", "&" + EL_INFO);
|
||||
String videoInfoPageString = downloader.download(videoInfoUrl);
|
||||
videoInfoPage = Parser.compatParseMap(videoInfoPageString);
|
||||
|
||||
// Check if the video is age restricted
|
||||
if (pageContent.contains("<meta property=\"og:restrictions:age")) {
|
||||
playerUrl = getPlayerUrlFromRestrictedVideo(getUrl());
|
||||
isAgeRestricted = true;
|
||||
} else {
|
||||
ytPlayerConfig = getPlayerConfig(pageContent);
|
||||
playerArgs = getPlayerArgs(ytPlayerConfig);
|
||||
playerUrl = getPlayerUrl(ytPlayerConfig);
|
||||
isAgeRestricted = false;
|
||||
}
|
||||
|
||||
if (decryptionCode.isEmpty()) {
|
||||
decryptionCode = loadDecryptionCode(playerUrl);
|
||||
}
|
||||
}
|
||||
|
||||
private JSONObject getPlayerConfig(String pageContent) throws ParsingException {
|
||||
try {
|
||||
String ytPlayerConfigRaw =
|
||||
Parser.matchGroup1("ytplayer.config\\s*=\\s*(\\{.*?\\});", pageContent);
|
||||
return new JSONObject(ytPlayerConfigRaw);
|
||||
} catch (Parser.RegexException e) {
|
||||
String errorReason = getErrorMessage();
|
||||
switch (errorReason) {
|
||||
case "GEMA":
|
||||
throw new GemaException(errorReason);
|
||||
case "":
|
||||
throw new ContentNotAvailableException("Content not available: player config empty", e);
|
||||
default:
|
||||
throw new ContentNotAvailableException("Content not available", e);
|
||||
}
|
||||
} catch (JSONException e) {
|
||||
throw new ParsingException("Could not parse yt player config", e);
|
||||
}
|
||||
}
|
||||
|
||||
private JSONObject getPlayerArgs(JSONObject playerConfig) throws ParsingException {
|
||||
JSONObject playerArgs;
|
||||
|
||||
//attempt to load the youtube js player JSON arguments
|
||||
boolean isLiveStream = false; //used to determine if this is a livestream or not
|
||||
try {
|
||||
playerArgs = playerConfig.getJSONObject("args");
|
||||
|
||||
// check if we have a live stream. We need to filter it, since its not yet supported.
|
||||
if ((playerArgs.has("ps") && playerArgs.get("ps").toString().equals("live"))
|
||||
|| (playerArgs.get(URL_ENCODED_FMT_STREAM_MAP).toString().isEmpty())) {
|
||||
isLiveStream = true;
|
||||
}
|
||||
} catch (JSONException e) {
|
||||
throw new ParsingException("Could not parse yt player config", e);
|
||||
}
|
||||
if (isLiveStream) {
|
||||
throw new LiveStreamException("This is a Life stream. Can't use those right now.");
|
||||
}
|
||||
|
||||
return playerArgs;
|
||||
}
|
||||
|
||||
private String getPlayerUrl(JSONObject playerConfig) throws ParsingException {
|
||||
try {
|
||||
// The Youtube service needs to be initialized by downloading the
|
||||
// js-Youtube-player. This is done in order to get the algorithm
|
||||
// for decrypting cryptic signatures inside certain stream urls.
|
||||
String playerUrl;
|
||||
|
||||
JSONObject ytAssets = playerConfig.getJSONObject("assets");
|
||||
playerUrl = ytAssets.getString("js");
|
||||
|
||||
if (playerUrl.startsWith("//")) {
|
||||
playerUrl = HTTPS + playerUrl;
|
||||
}
|
||||
return playerUrl;
|
||||
} catch (JSONException e) {
|
||||
throw new ParsingException(
|
||||
"Could not load decryption code for the Youtube service.", e);
|
||||
}
|
||||
}
|
||||
|
||||
private String getPlayerUrlFromRestrictedVideo(String pageUrl) throws ParsingException, ReCaptchaException {
|
||||
try {
|
||||
Downloader downloader = NewPipe.getDownloader();
|
||||
String playerUrl = "";
|
||||
String videoId = getUrlIdHandler().getId(pageUrl);
|
||||
String embedUrl = "https://www.youtube.com/embed/" + videoId;
|
||||
String embedPageContent = downloader.download(embedUrl);
|
||||
//todo: find out if this can be reapaced by Parser.matchGroup1()
|
||||
Pattern assetsPattern = Pattern.compile("\"assets\":.+?\"js\":\\s*(\"[^\"]+\")");
|
||||
Matcher patternMatcher = assetsPattern.matcher(embedPageContent);
|
||||
while (patternMatcher.find()) {
|
||||
playerUrl = patternMatcher.group(1);
|
||||
}
|
||||
playerUrl = playerUrl.replace("\\", "").replace("\"", "");
|
||||
|
||||
if (playerUrl.startsWith("//")) {
|
||||
playerUrl = HTTPS + playerUrl;
|
||||
}
|
||||
return playerUrl;
|
||||
} catch (IOException e) {
|
||||
throw new ParsingException(
|
||||
"Could load decryption code form restricted video for the Youtube service.", e);
|
||||
} catch (ReCaptchaException e) {
|
||||
throw new ReCaptchaException("reCaptcha Challenge requested");
|
||||
}
|
||||
}
|
||||
|
||||
private String loadDecryptionCode(String playerUrl) throws DecryptException {
|
||||
String decryptionFuncName;
|
||||
@ -935,8 +780,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
||||
return decryptionCode;
|
||||
}
|
||||
|
||||
private String decryptSignature(String encryptedSig, String decryptionCode)
|
||||
throws DecryptException {
|
||||
private String decryptSignature(String encryptedSig, String decryptionCode) throws DecryptException {
|
||||
Context context = Context.enter();
|
||||
context.setOptimizationLevel(-1);
|
||||
Object result = null;
|
||||
@ -953,28 +797,84 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
||||
return result == null ? "" : result.toString();
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* {@inheritDoc}
|
||||
* Provides information about links to other videos on the video page, such as related videos.
|
||||
* This is encapsulated in a StreamInfoItem object,
|
||||
* which is a subset of the fields in a full StreamInfo.
|
||||
*/
|
||||
public String getErrorMessage() {
|
||||
String errorMessage = doc.select("h1[id=\"unavailable-message\"]").first().text();
|
||||
StringBuilder errorReason;
|
||||
private StreamInfoItemExtractor extractVideoPreviewInfo(final Element li) {
|
||||
return new StreamInfoItemExtractor() {
|
||||
@Override
|
||||
public StreamType getStreamType() throws ParsingException {
|
||||
return StreamType.VIDEO_STREAM;
|
||||
}
|
||||
|
||||
if (errorMessage == null || errorMessage.isEmpty()) {
|
||||
errorReason = null;
|
||||
} else if(errorMessage.contains("GEMA")) {
|
||||
// Gema sometimes blocks youtube music content in germany:
|
||||
// https://www.gema.de/en/
|
||||
// Detailed description:
|
||||
// https://en.wikipedia.org/wiki/GEMA_%28German_organization%29
|
||||
errorReason = new StringBuilder("GEMA");
|
||||
} else {
|
||||
errorReason = new StringBuilder(errorMessage);
|
||||
errorReason.append(" ");
|
||||
errorReason.append(doc.select("[id=\"unavailable-submessage\"]").first().text());
|
||||
}
|
||||
@Override
|
||||
public boolean isAd() throws ParsingException {
|
||||
return !li.select("span[class*=\"icon-not-available\"]").isEmpty();
|
||||
}
|
||||
|
||||
return errorReason != null ? errorReason.toString() : null;
|
||||
@Override
|
||||
public String getWebPageUrl() throws ParsingException {
|
||||
return li.select("a.content-link").first().attr("abs:href");
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getTitle() throws ParsingException {
|
||||
//todo: check NullPointerException causing
|
||||
return li.select("span.title").first().text();
|
||||
//this page causes the NullPointerException, after finding it by searching for "tjvg":
|
||||
//https://www.youtube.com/watch?v=Uqg0aEhLFAg
|
||||
}
|
||||
|
||||
@Override
|
||||
public int getDuration() throws ParsingException {
|
||||
return YoutubeParsingHelper.parseDurationString(
|
||||
li.select("span.video-time").first().text());
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getUploader() throws ParsingException {
|
||||
return li.select("span.g-hovercard").first().text();
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getUploadDate() throws ParsingException {
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getViewCount() throws ParsingException {
|
||||
//this line is unused
|
||||
//String views = li.select("span.view-count").first().text();
|
||||
|
||||
//Log.i(TAG, "title:"+info.title);
|
||||
//Log.i(TAG, "view count:"+views);
|
||||
|
||||
try {
|
||||
return Long.parseLong(Utils.removeNonDigitCharacters(
|
||||
li.select("span.view-count").first().text()));
|
||||
} catch (Exception e) {
|
||||
//related videos sometimes have no view count
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getThumbnailUrl() throws ParsingException {
|
||||
Element img = li.select("img").first();
|
||||
String thumbnailUrl = img.attr("abs:src");
|
||||
// Sometimes youtube sends links to gif files which somehow seem to not exist
|
||||
// anymore. Items with such gif also offer a secondary image source. So we are going
|
||||
// to use that if we caught such an item.
|
||||
if (thumbnailUrl.contains(".gif")) {
|
||||
thumbnailUrl = img.attr("data-thumb");
|
||||
}
|
||||
if (thumbnailUrl.startsWith("//")) {
|
||||
thumbnailUrl = HTTPS + thumbnailUrl;
|
||||
}
|
||||
return thumbnailUrl;
|
||||
}
|
||||
};
|
||||
}
|
||||
}
|
||||
|
@ -3,8 +3,9 @@ package org.schabi.newpipe.extractor.services.youtube;
|
||||
import org.jsoup.nodes.Element;
|
||||
import org.schabi.newpipe.extractor.exceptions.FoundAdException;
|
||||
import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
||||
import org.schabi.newpipe.extractor.stream.AbstractStreamInfo;
|
||||
import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor;
|
||||
import org.schabi.newpipe.extractor.stream.StreamType;
|
||||
import org.schabi.newpipe.extractor.utils.Utils;
|
||||
|
||||
/*
|
||||
* Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
|
||||
@ -116,7 +117,7 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
|
||||
}
|
||||
}
|
||||
|
||||
output = input.replaceAll("[^0-9]+", "");
|
||||
output = Utils.removeNonDigitCharacters(input);
|
||||
|
||||
try {
|
||||
return Long.parseLong(output);
|
||||
@ -150,11 +151,11 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
|
||||
}
|
||||
|
||||
@Override
|
||||
public AbstractStreamInfo.StreamType getStreamType() {
|
||||
public StreamType getStreamType() {
|
||||
if (isLiveStream(item)) {
|
||||
return AbstractStreamInfo.StreamType.LIVE_STREAM;
|
||||
return StreamType.LIVE_STREAM;
|
||||
} else {
|
||||
return AbstractStreamInfo.StreamType.VIDEO_STREAM;
|
||||
return StreamType.VIDEO_STREAM;
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -140,6 +140,7 @@ public class YoutubeStreamUrlIdHandler implements UrlIdHandler {
|
||||
return Parser.matchGroup1("ci=" + ID_PATTERN, uri.getQuery());
|
||||
}
|
||||
|
||||
@Override
|
||||
public String cleanUrl(String complexUrl) throws ParsingException {
|
||||
return getUrl(getId(complexUrl));
|
||||
}
|
||||
|
@ -1,41 +0,0 @@
|
||||
package org.schabi.newpipe.extractor.stream;
|
||||
|
||||
/*
|
||||
* Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
|
||||
* AbstractStreamInfo.java is part of NewPipe.
|
||||
*
|
||||
* NewPipe is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* NewPipe is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
import org.schabi.newpipe.extractor.Info;
|
||||
|
||||
/**
|
||||
* Common properties between StreamInfo and StreamInfoItem.
|
||||
*/
|
||||
public abstract class AbstractStreamInfo extends Info {
|
||||
public enum StreamType {
|
||||
NONE, // placeholder to check if stream type was checked or not
|
||||
VIDEO_STREAM,
|
||||
AUDIO_STREAM,
|
||||
LIVE_STREAM,
|
||||
AUDIO_LIVE_STREAM,
|
||||
FILE
|
||||
}
|
||||
|
||||
public StreamType stream_type;
|
||||
public String uploader = "";
|
||||
public String thumbnail_url = "";
|
||||
public String upload_date = "";
|
||||
public long view_count = -1;
|
||||
}
|
@ -1,7 +1,5 @@
|
||||
package org.schabi.newpipe.extractor.stream;
|
||||
|
||||
import java.io.Serializable;
|
||||
|
||||
/*
|
||||
* Created by Christian Schabesberger on 04.03.16.
|
||||
*
|
||||
@ -22,31 +20,17 @@ import java.io.Serializable;
|
||||
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
public class AudioStream implements Serializable {
|
||||
public String url = "";
|
||||
public int format = -1;
|
||||
public int bandwidth = -1;
|
||||
public int sampling_rate = -1;
|
||||
public int avgBitrate = -1;
|
||||
public class AudioStream extends Stream {
|
||||
public int average_bitrate = -1;
|
||||
|
||||
public AudioStream(String url, int format, int avgBitrate, int bandwidth, int samplingRate) {
|
||||
this.url = url;
|
||||
this.format = format;
|
||||
this.avgBitrate = avgBitrate;
|
||||
this.bandwidth = bandwidth;
|
||||
this.sampling_rate = samplingRate;
|
||||
public AudioStream(String url, int format, int averageBitrate) {
|
||||
super(url, format);
|
||||
this.average_bitrate = averageBitrate;
|
||||
}
|
||||
|
||||
// reveals whether two streams are the same, but have different urls
|
||||
public boolean equalStats(AudioStream cmp) {
|
||||
return format == cmp.format
|
||||
&& bandwidth == cmp.bandwidth
|
||||
&& sampling_rate == cmp.sampling_rate
|
||||
&& avgBitrate == cmp.avgBitrate;
|
||||
}
|
||||
|
||||
// reveals whether two streams are equal
|
||||
public boolean equals(AudioStream cmp) {
|
||||
return cmp != null && equalStats(cmp) && url.equals(cmp.url);
|
||||
@Override
|
||||
public boolean equalStats(Stream cmp) {
|
||||
return super.equalStats(cmp) && cmp instanceof AudioStream &&
|
||||
average_bitrate == ((AudioStream) cmp).average_bitrate;
|
||||
}
|
||||
}
|
||||
|
39
stream/Stream.java
Normal file
39
stream/Stream.java
Normal file
@ -0,0 +1,39 @@
|
||||
package org.schabi.newpipe.extractor.stream;
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.util.List;
|
||||
|
||||
public abstract class Stream implements Serializable {
|
||||
public String url;
|
||||
public int format = -1;
|
||||
|
||||
public Stream(String url, int format) {
|
||||
this.url = url;
|
||||
this.format = format;
|
||||
}
|
||||
|
||||
/**
|
||||
* Reveals whether two streams are the same, but have different urls
|
||||
*/
|
||||
public boolean equalStats(Stream cmp) {
|
||||
return cmp != null && format == cmp.format;
|
||||
}
|
||||
|
||||
/**
|
||||
* Reveals whether two Streams are equal
|
||||
*/
|
||||
public boolean equals(Stream cmp) {
|
||||
return equalStats(cmp) && url.equals(cmp.url);
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if the list already contains one stream with equals stats
|
||||
*/
|
||||
public static boolean containSimilarStream(Stream stream, List<? extends Stream> streamList) {
|
||||
if (stream == null || streamList == null) return false;
|
||||
for (Stream cmpStream : streamList) {
|
||||
if (stream.equalStats(cmpStream)) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
}
|
@ -31,20 +31,11 @@ import java.util.List;
|
||||
*/
|
||||
public abstract class StreamExtractor extends Extractor {
|
||||
|
||||
public static class ContentNotAvailableException extends ParsingException {
|
||||
public ContentNotAvailableException(String message) {
|
||||
super(message);
|
||||
}
|
||||
|
||||
public ContentNotAvailableException(String message, Throwable cause) {
|
||||
super(message, cause);
|
||||
}
|
||||
}
|
||||
|
||||
public StreamExtractor(UrlIdHandler urlIdHandler, String url, int serviceId) {
|
||||
super(urlIdHandler, serviceId, url);
|
||||
}
|
||||
|
||||
public abstract String getId() throws ParsingException;
|
||||
public abstract int getTimeStamp() throws ParsingException;
|
||||
public abstract String getTitle() throws ParsingException;
|
||||
public abstract String getDescription() throws ParsingException;
|
||||
@ -65,8 +56,7 @@ public abstract class StreamExtractor extends Extractor {
|
||||
public abstract int getDislikeCount() throws ParsingException;
|
||||
public abstract StreamInfoItemExtractor getNextVideo() throws ParsingException;
|
||||
public abstract StreamInfoItemCollector getRelatedVideos() throws ParsingException;
|
||||
public abstract String getPageUrl();
|
||||
public abstract StreamInfo.StreamType getStreamType() throws ParsingException;
|
||||
public abstract StreamType getStreamType() throws ParsingException;
|
||||
|
||||
/**
|
||||
* Analyses the webpage's document and extracts any error message there might be.
|
||||
|
@ -1,7 +1,8 @@
|
||||
package org.schabi.newpipe.extractor.stream;
|
||||
|
||||
import org.schabi.newpipe.extractor.Info;
|
||||
import org.schabi.newpipe.extractor.InfoItem;
|
||||
import org.schabi.newpipe.extractor.UrlIdHandler;
|
||||
import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
|
||||
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
|
||||
import org.schabi.newpipe.extractor.utils.DashMpdParser;
|
||||
|
||||
@ -31,11 +32,11 @@ import java.util.Vector;
|
||||
/**
|
||||
* Info object for opened videos, ie the video ready to play.
|
||||
*/
|
||||
@SuppressWarnings("ALL")
|
||||
public class StreamInfo extends AbstractStreamInfo {
|
||||
@SuppressWarnings("WeakerAccess")
|
||||
public class StreamInfo extends Info {
|
||||
|
||||
public static class StreamExctractException extends ExtractionException {
|
||||
StreamExctractException(String message) {
|
||||
public static class StreamExtractException extends ExtractionException {
|
||||
StreamExtractException(String message) {
|
||||
super(message);
|
||||
}
|
||||
}
|
||||
@ -43,43 +44,11 @@ public class StreamInfo extends AbstractStreamInfo {
|
||||
public StreamInfo() {
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a new StreamInfo object from an existing AbstractVideoInfo.
|
||||
* All the shared properties are copied to the new StreamInfo.
|
||||
*/
|
||||
@SuppressWarnings("WeakerAccess")
|
||||
public StreamInfo(AbstractStreamInfo avi) {
|
||||
this.id = avi.id;
|
||||
this.url = avi.url;
|
||||
this.name = avi.name;
|
||||
this.uploader = avi.uploader;
|
||||
this.thumbnail_url = avi.thumbnail_url;
|
||||
this.upload_date = avi.upload_date;
|
||||
this.upload_date = avi.upload_date;
|
||||
this.view_count = avi.view_count;
|
||||
|
||||
//todo: better than this
|
||||
if (avi instanceof StreamInfoItem) {
|
||||
//shitty String to convert code
|
||||
/*
|
||||
String dur = ((StreamInfoItem)avi).duration;
|
||||
int minutes = Integer.parseInt(dur.substring(0, dur.indexOf(":")));
|
||||
int seconds = Integer.parseInt(dur.substring(dur.indexOf(":")+1, dur.length()));
|
||||
*/
|
||||
this.duration = ((StreamInfoItem) avi).duration;
|
||||
}
|
||||
}
|
||||
|
||||
public void addException(Exception e) {
|
||||
errors.add(e);
|
||||
}
|
||||
|
||||
/**
|
||||
* Fills out the video info fields which are common to all services.
|
||||
* Probably needs to be overridden by subclasses
|
||||
*/
|
||||
public static StreamInfo getVideoInfo(StreamExtractor extractor)
|
||||
throws ExtractionException, StreamExtractor.ContentNotAvailableException {
|
||||
public static StreamInfo getVideoInfo(StreamExtractor extractor) throws ExtractionException {
|
||||
StreamInfo streamInfo = new StreamInfo();
|
||||
|
||||
try {
|
||||
@ -87,15 +56,15 @@ public class StreamInfo extends AbstractStreamInfo {
|
||||
streamInfo = extractStreams(streamInfo, extractor);
|
||||
streamInfo = extractOptionalData(streamInfo, extractor);
|
||||
} catch (ExtractionException e) {
|
||||
// Currently YouTube does not distinguish between age restricted videos and videos blocked
|
||||
// by country. This means that during the initialisation of the extractor, the extractor
|
||||
// will assume that a video is age restricted while in reality it it blocked by country.
|
||||
//
|
||||
// We will now detect whether the video is blocked by country or not.
|
||||
// Currently YouTube does not distinguish between age restricted videos and videos blocked
|
||||
// by country. This means that during the initialisation of the extractor, the extractor
|
||||
// will assume that a video is age restricted while in reality it it blocked by country.
|
||||
//
|
||||
// We will now detect whether the video is blocked by country or not.
|
||||
String errorMsg = extractor.getErrorMessage();
|
||||
|
||||
if (errorMsg != null) {
|
||||
throw new StreamExtractor.ContentNotAvailableException(errorMsg);
|
||||
throw new ContentNotAvailableException(errorMsg);
|
||||
} else {
|
||||
throw e;
|
||||
}
|
||||
@ -104,18 +73,14 @@ public class StreamInfo extends AbstractStreamInfo {
|
||||
return streamInfo;
|
||||
}
|
||||
|
||||
private static StreamInfo extractImportantData(
|
||||
StreamInfo streamInfo, StreamExtractor extractor)
|
||||
throws ExtractionException {
|
||||
private static StreamInfo extractImportantData(StreamInfo streamInfo, StreamExtractor extractor) throws ExtractionException {
|
||||
/* ---- important data, withoug the video can't be displayed goes here: ---- */
|
||||
// if one of these is not available an exception is meant to be thrown directly into the frontend.
|
||||
|
||||
UrlIdHandler uiconv = extractor.getUrlIdHandler();
|
||||
|
||||
streamInfo.service_id = extractor.getServiceId();
|
||||
streamInfo.url = extractor.getPageUrl();
|
||||
streamInfo.url = extractor.getUrl();
|
||||
streamInfo.stream_type = extractor.getStreamType();
|
||||
streamInfo.id = uiconv.getId(extractor.getPageUrl());
|
||||
streamInfo.id = extractor.getId();
|
||||
streamInfo.name = extractor.getTitle();
|
||||
streamInfo.age_limit = extractor.getAgeLimit();
|
||||
|
||||
@ -130,9 +95,7 @@ public class StreamInfo extends AbstractStreamInfo {
|
||||
return streamInfo;
|
||||
}
|
||||
|
||||
private static StreamInfo extractStreams(
|
||||
StreamInfo streamInfo, StreamExtractor extractor)
|
||||
throws ExtractionException {
|
||||
private static StreamInfo extractStreams(StreamInfo streamInfo, StreamExtractor extractor) throws ExtractionException {
|
||||
/* ---- stream extraction goes here ---- */
|
||||
// At least one type of stream has to be available,
|
||||
// otherwise an exception will be thrown directly into the frontend.
|
||||
@ -149,34 +112,33 @@ public class StreamInfo extends AbstractStreamInfo {
|
||||
} catch (Exception e) {
|
||||
streamInfo.addException(new ExtractionException("Couldn't get audio streams", e));
|
||||
}
|
||||
// also try to get streams from the dashMpd
|
||||
if (streamInfo.dashMpdUrl != null && !streamInfo.dashMpdUrl.isEmpty()) {
|
||||
if (streamInfo.audio_streams == null) {
|
||||
streamInfo.audio_streams = new Vector<>();
|
||||
}
|
||||
//todo: make this quick and dirty solution a real fallback
|
||||
// same as the quick and dirty above
|
||||
try {
|
||||
streamInfo.audio_streams.addAll(
|
||||
DashMpdParser.getAudioStreams(streamInfo.dashMpdUrl));
|
||||
} catch (Exception e) {
|
||||
streamInfo.addException(
|
||||
new ExtractionException("Couldn't get audio streams from dash mpd", e));
|
||||
}
|
||||
}
|
||||
/* Extract video stream url*/
|
||||
try {
|
||||
streamInfo.video_streams = extractor.getVideoStreams();
|
||||
} catch (Exception e) {
|
||||
streamInfo.addException(
|
||||
new ExtractionException("Couldn't get video streams", e));
|
||||
streamInfo.addException(new ExtractionException("Couldn't get video streams", e));
|
||||
}
|
||||
/* Extract video only stream url*/
|
||||
try {
|
||||
streamInfo.video_only_streams = extractor.getVideoOnlyStreams();
|
||||
} catch (Exception e) {
|
||||
streamInfo.addException(
|
||||
new ExtractionException("Couldn't get video only streams", e));
|
||||
streamInfo.addException(new ExtractionException("Couldn't get video only streams", e));
|
||||
}
|
||||
|
||||
// Lists can be null if a exception was thrown during extraction
|
||||
if (streamInfo.video_streams == null) streamInfo.video_streams = new Vector<>();
|
||||
if (streamInfo.video_only_streams == null) streamInfo.video_only_streams = new Vector<>();
|
||||
if (streamInfo.audio_streams == null) streamInfo.audio_streams = new Vector<>();
|
||||
|
||||
if (streamInfo.dashMpdUrl != null && !streamInfo.dashMpdUrl.isEmpty()) {
|
||||
try {
|
||||
// Will try to find in the dash manifest for any stream that the ItagItem has (by the id),
|
||||
// it has video, video only and audio streams and will only add to the list if it don't
|
||||
// find a similar stream in the respective lists (calling Stream#equalStats).
|
||||
DashMpdParser.getStreams(streamInfo);
|
||||
} catch (Exception e) {
|
||||
streamInfo.addException(new ExtractionException("Couldn't get streams from dash mpd", e));
|
||||
}
|
||||
}
|
||||
|
||||
// either dash_mpd audio_only or video has to be available, otherwise we didn't get a stream,
|
||||
@ -184,15 +146,14 @@ public class StreamInfo extends AbstractStreamInfo {
|
||||
if ((streamInfo.video_streams == null || streamInfo.video_streams.isEmpty())
|
||||
&& (streamInfo.audio_streams == null || streamInfo.audio_streams.isEmpty())
|
||||
&& (streamInfo.dashMpdUrl == null || streamInfo.dashMpdUrl.isEmpty())) {
|
||||
throw new StreamExctractException(
|
||||
throw new StreamExtractException(
|
||||
"Could not get any stream. See error variable to get further details.");
|
||||
}
|
||||
|
||||
return streamInfo;
|
||||
}
|
||||
|
||||
private static StreamInfo extractOptionalData(
|
||||
StreamInfo streamInfo, StreamExtractor extractor) {
|
||||
private static StreamInfo extractOptionalData(StreamInfo streamInfo, StreamExtractor extractor) {
|
||||
/* ---- optional data goes here: ---- */
|
||||
// If one of these fails, the frontend needs to handle that they are not available.
|
||||
// Exceptions are therefore not thrown into the frontend, but stored into the error List,
|
||||
@ -259,8 +220,7 @@ public class StreamInfo extends AbstractStreamInfo {
|
||||
streamInfo.addException(e);
|
||||
}
|
||||
try {
|
||||
StreamInfoItemCollector c = new StreamInfoItemCollector(
|
||||
extractor.getUrlIdHandler(), extractor.getServiceId());
|
||||
StreamInfoItemCollector c = new StreamInfoItemCollector(extractor.getServiceId());
|
||||
StreamInfoItemExtractor nextVideo = extractor.getNextVideo();
|
||||
c.commit(nextVideo);
|
||||
if (c.getItemList().size() != 0) {
|
||||
@ -282,26 +242,36 @@ public class StreamInfo extends AbstractStreamInfo {
|
||||
return streamInfo;
|
||||
}
|
||||
|
||||
public String uploader_thumbnail_url = "";
|
||||
public String channel_url = "";
|
||||
public String description = "";
|
||||
public void addException(Exception e) {
|
||||
errors.add(e);
|
||||
}
|
||||
|
||||
public List<VideoStream> video_streams = null;
|
||||
public List<AudioStream> audio_streams = null;
|
||||
public List<VideoStream> video_only_streams = null;
|
||||
public StreamType stream_type;
|
||||
public String uploader;
|
||||
public String thumbnail_url;
|
||||
public String upload_date;
|
||||
public long view_count = -1;
|
||||
|
||||
public String uploader_thumbnail_url;
|
||||
public String channel_url;
|
||||
public String description;
|
||||
|
||||
public List<VideoStream> video_streams;
|
||||
public List<AudioStream> audio_streams;
|
||||
public List<VideoStream> video_only_streams;
|
||||
// video streams provided by the dash mpd do not need to be provided as VideoStream.
|
||||
// Later on this will also aplly to audio streams. Since dash mpd is standarized,
|
||||
// crawling such a file is not service dependent. Therefore getting audio only streams by yust
|
||||
// providing the dash mpd fille will be possible in the future.
|
||||
public String dashMpdUrl = "";
|
||||
public String dashMpdUrl;
|
||||
public int duration = -1;
|
||||
|
||||
public int age_limit = -1;
|
||||
public int like_count = -1;
|
||||
public int dislike_count = -1;
|
||||
public String average_rating = "";
|
||||
public StreamInfoItem next_video = null;
|
||||
public List<InfoItem> related_streams = null;
|
||||
public String average_rating;
|
||||
public StreamInfoItem next_video;
|
||||
public List<InfoItem> related_streams = new Vector<>();
|
||||
//in seconds. some metadata is not passed using a StreamInfo object!
|
||||
public int start_position = 0;
|
||||
}
|
||||
|
@ -25,18 +25,16 @@ import org.schabi.newpipe.extractor.InfoItem;
|
||||
/**
|
||||
* Info object for previews of unopened videos, eg search results, related videos
|
||||
*/
|
||||
public class StreamInfoItem extends AbstractStreamInfo implements InfoItem {
|
||||
public int duration;
|
||||
public class StreamInfoItem extends InfoItem {
|
||||
public StreamType stream_type;
|
||||
|
||||
public InfoType infoType() {
|
||||
return InfoType.STREAM;
|
||||
}
|
||||
public String uploader;
|
||||
public String thumbnail_url;
|
||||
public String upload_date;
|
||||
public long view_count = -1;
|
||||
public int duration = -1;
|
||||
|
||||
public String getTitle() {
|
||||
return name;
|
||||
}
|
||||
|
||||
public String getLink() {
|
||||
return url;
|
||||
public StreamInfoItem() {
|
||||
super(InfoType.STREAM);
|
||||
}
|
||||
}
|
@ -1,8 +1,6 @@
|
||||
package org.schabi.newpipe.extractor.stream;
|
||||
|
||||
import org.schabi.newpipe.extractor.InfoItemCollector;
|
||||
import org.schabi.newpipe.extractor.NewPipe;
|
||||
import org.schabi.newpipe.extractor.UrlIdHandler;
|
||||
import org.schabi.newpipe.extractor.exceptions.FoundAdException;
|
||||
import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
||||
|
||||
@ -28,15 +26,8 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
||||
|
||||
public class StreamInfoItemCollector extends InfoItemCollector {
|
||||
|
||||
private UrlIdHandler urlIdHandler;
|
||||
|
||||
public StreamInfoItemCollector(UrlIdHandler handler, int serviceId) {
|
||||
public StreamInfoItemCollector(int serviceId) {
|
||||
super(serviceId);
|
||||
urlIdHandler = handler;
|
||||
}
|
||||
|
||||
private UrlIdHandler getUrlIdHandler() {
|
||||
return urlIdHandler;
|
||||
}
|
||||
|
||||
public StreamInfoItem extract(StreamInfoItemExtractor extractor) throws Exception {
|
||||
@ -48,13 +39,7 @@ public class StreamInfoItemCollector extends InfoItemCollector {
|
||||
// important information
|
||||
resultItem.service_id = getServiceId();
|
||||
resultItem.url = extractor.getWebPageUrl();
|
||||
if (getUrlIdHandler() == null) {
|
||||
throw new ParsingException("Error: UrlIdHandler not set");
|
||||
} else if (!resultItem.url.isEmpty()) {
|
||||
resultItem.id = NewPipe.getService(getServiceId())
|
||||
.getStreamUrlIdHandlerInstance()
|
||||
.getId(resultItem.url);
|
||||
}
|
||||
|
||||
resultItem.name = extractor.getTitle();
|
||||
resultItem.stream_type = extractor.getStreamType();
|
||||
|
||||
|
@ -23,7 +23,7 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
||||
*/
|
||||
|
||||
public interface StreamInfoItemExtractor {
|
||||
AbstractStreamInfo.StreamType getStreamType() throws ParsingException;
|
||||
StreamType getStreamType() throws ParsingException;
|
||||
String getWebPageUrl() throws ParsingException;
|
||||
String getTitle() throws ParsingException;
|
||||
int getDuration() throws ParsingException;
|
||||
|
10
stream/StreamType.java
Normal file
10
stream/StreamType.java
Normal file
@ -0,0 +1,10 @@
|
||||
package org.schabi.newpipe.extractor.stream;
|
||||
|
||||
public enum StreamType {
|
||||
NONE, // placeholder to check if stream type was checked or not
|
||||
VIDEO_STREAM,
|
||||
AUDIO_STREAM,
|
||||
LIVE_STREAM,
|
||||
AUDIO_LIVE_STREAM,
|
||||
FILE
|
||||
}
|
@ -1,7 +1,5 @@
|
||||
package org.schabi.newpipe.extractor.stream;
|
||||
|
||||
import java.io.Serializable;
|
||||
|
||||
/*
|
||||
* Created by Christian Schabesberger on 04.03.16.
|
||||
*
|
||||
@ -22,31 +20,24 @@ import java.io.Serializable;
|
||||
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
public class VideoStream implements Serializable {
|
||||
//url of the stream
|
||||
public String url = "";
|
||||
public int format = -1;
|
||||
public String resolution = "";
|
||||
public boolean isVideoOnly = false;
|
||||
public class VideoStream extends Stream {
|
||||
public String resolution;
|
||||
public boolean isVideoOnly;
|
||||
|
||||
public VideoStream(String url, int format, String res) {
|
||||
this(false, url, format, res);
|
||||
this(url, format, res, false);
|
||||
}
|
||||
|
||||
public VideoStream(boolean isVideoOnly, String url, int format, String res) {
|
||||
this.url = url;
|
||||
this.format = format;
|
||||
public VideoStream(String url, int format, String res, boolean isVideoOnly) {
|
||||
super(url, format);
|
||||
this.resolution = res;
|
||||
this.isVideoOnly = isVideoOnly;
|
||||
}
|
||||
|
||||
// reveals whether two streams are the same, but have different urls
|
||||
public boolean equalStats(VideoStream cmp) {
|
||||
return format == cmp.format && resolution.equals(cmp.resolution);
|
||||
}
|
||||
|
||||
// reveals whether two streams are equal
|
||||
public boolean equals(VideoStream cmp) {
|
||||
return cmp != null && equalStats(cmp) && url.equals(cmp.url);
|
||||
@Override
|
||||
public boolean equalStats(Stream cmp) {
|
||||
return super.equalStats(cmp) && cmp instanceof VideoStream &&
|
||||
resolution.equals(((VideoStream) cmp).resolution) &&
|
||||
isVideoOnly == ((VideoStream) cmp).isVideoOnly;
|
||||
}
|
||||
}
|
||||
|
@ -5,7 +5,11 @@ import org.schabi.newpipe.extractor.MediaFormat;
|
||||
import org.schabi.newpipe.extractor.NewPipe;
|
||||
import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
||||
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
|
||||
import org.schabi.newpipe.extractor.services.youtube.ItagItem;
|
||||
import org.schabi.newpipe.extractor.stream.AudioStream;
|
||||
import org.schabi.newpipe.extractor.stream.Stream;
|
||||
import org.schabi.newpipe.extractor.stream.StreamInfo;
|
||||
import org.schabi.newpipe.extractor.stream.VideoStream;
|
||||
import org.w3c.dom.Document;
|
||||
import org.w3c.dom.Element;
|
||||
import org.w3c.dom.NodeList;
|
||||
@ -13,8 +17,6 @@ import org.w3c.dom.NodeList;
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.util.List;
|
||||
import java.util.Vector;
|
||||
|
||||
import javax.xml.parsers.DocumentBuilder;
|
||||
import javax.xml.parsers.DocumentBuilderFactory;
|
||||
@ -44,24 +46,25 @@ public class DashMpdParser {
|
||||
private DashMpdParser() {
|
||||
}
|
||||
|
||||
static class DashMpdParsingException extends ParsingException {
|
||||
public static class DashMpdParsingException extends ParsingException {
|
||||
DashMpdParsingException(String message, Exception e) {
|
||||
super(message, e);
|
||||
}
|
||||
}
|
||||
|
||||
public static List<AudioStream> getAudioStreams(String dashManifestUrl)
|
||||
throws DashMpdParsingException, ReCaptchaException {
|
||||
/**
|
||||
* Download manifest and return nodelist with elements of tag "AdaptationSet"
|
||||
*/
|
||||
public static void getStreams(StreamInfo streamInfo) throws DashMpdParsingException, ReCaptchaException {
|
||||
String dashDoc;
|
||||
Downloader downloader = NewPipe.getDownloader();
|
||||
try {
|
||||
dashDoc = downloader.download(dashManifestUrl);
|
||||
dashDoc = downloader.download(streamInfo.dashMpdUrl);
|
||||
} catch (IOException ioe) {
|
||||
throw new DashMpdParsingException("Could not get dash mpd: " + dashManifestUrl, ioe);
|
||||
throw new DashMpdParsingException("Could not get dash mpd: " + streamInfo.dashMpdUrl, ioe);
|
||||
} catch (ReCaptchaException e) {
|
||||
throw new ReCaptchaException("reCaptcha Challenge needed");
|
||||
}
|
||||
Vector<AudioStream> audioStreams = new Vector<>();
|
||||
|
||||
try {
|
||||
DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
|
||||
@ -69,27 +72,43 @@ public class DashMpdParser {
|
||||
InputStream stream = new ByteArrayInputStream(dashDoc.getBytes());
|
||||
|
||||
Document doc = builder.parse(stream);
|
||||
NodeList adaptationSetList = doc.getElementsByTagName("AdaptationSet");
|
||||
for (int i = 0; i < adaptationSetList.getLength(); i++) {
|
||||
Element adaptationSet = (Element) adaptationSetList.item(i);
|
||||
String memeType = adaptationSet.getAttribute("mimeType");
|
||||
if (memeType.contains("audio")) {
|
||||
Element representation = (Element) adaptationSet.getElementsByTagName("Representation").item(0);
|
||||
NodeList representationList = doc.getElementsByTagName("Representation");
|
||||
|
||||
for (int i = 0; i < representationList.getLength(); i++) {
|
||||
Element representation = ((Element) representationList.item(i));
|
||||
try {
|
||||
String mimeType = ((Element) representation.getParentNode()).getAttribute("mimeType");
|
||||
String id = representation.getAttribute("id");
|
||||
String url = representation.getElementsByTagName("BaseURL").item(0).getTextContent();
|
||||
int bandwidth = Integer.parseInt(representation.getAttribute("bandwidth"));
|
||||
int samplingRate = Integer.parseInt(representation.getAttribute("audioSamplingRate"));
|
||||
int format = -1;
|
||||
if (memeType.equals(MediaFormat.WEBMA.mimeType)) {
|
||||
format = MediaFormat.WEBMA.id;
|
||||
} else if (memeType.equals(MediaFormat.M4A.mimeType)) {
|
||||
format = MediaFormat.M4A.id;
|
||||
ItagItem itag = ItagItem.getItag(Integer.parseInt(id));
|
||||
if (itag != null) {
|
||||
MediaFormat mediaFormat = MediaFormat.getFromMimeType(mimeType);
|
||||
int format = mediaFormat != null ? mediaFormat.id : -1;
|
||||
|
||||
if (itag.itagType.equals(ItagItem.ItagType.AUDIO)) {
|
||||
AudioStream audioStream = new AudioStream(url, format, itag.avgBitrate);
|
||||
|
||||
if (!Stream.containSimilarStream(audioStream, streamInfo.audio_streams)) {
|
||||
streamInfo.audio_streams.add(audioStream);
|
||||
}
|
||||
} else {
|
||||
boolean isVideoOnly = itag.itagType.equals(ItagItem.ItagType.VIDEO_ONLY);
|
||||
VideoStream videoStream = new VideoStream(url, format, itag.resolutionString, isVideoOnly);
|
||||
|
||||
if (isVideoOnly) {
|
||||
if (!Stream.containSimilarStream(videoStream, streamInfo.video_only_streams)) {
|
||||
streamInfo.video_only_streams.add(videoStream);
|
||||
}
|
||||
} else if (!Stream.containSimilarStream(videoStream, streamInfo.video_streams)) {
|
||||
streamInfo.video_streams.add(videoStream);
|
||||
}
|
||||
}
|
||||
}
|
||||
audioStreams.add(new AudioStream(url, format, 0, bandwidth, samplingRate));
|
||||
} catch (Exception ignored) {
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
throw new DashMpdParsingException("Could not parse Dash mpd", e);
|
||||
}
|
||||
return audioStreams;
|
||||
}
|
||||
}
|
||||
|
20
utils/Utils.java
Normal file
20
utils/Utils.java
Normal file
@ -0,0 +1,20 @@
|
||||
package org.schabi.newpipe.extractor.utils;
|
||||
|
||||
public class Utils {
|
||||
private Utils() {
|
||||
//no instance
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove all non-digit characters from a string.<p>
|
||||
* Examples:<br/>
|
||||
* <ul><li>1 234 567 views -> 1234567</li>
|
||||
* <li>$ 31,133.124 -> 31133124</li></ul>
|
||||
*
|
||||
* @param toRemove string to remove non-digit chars
|
||||
* @return a string that contains only digits
|
||||
*/
|
||||
public static String removeNonDigitCharacters(String toRemove) {
|
||||
return toRemove.replaceAll("\\D+", "");
|
||||
}
|
||||
}
|
Loading…
Reference in New Issue
Block a user