mirror of
https://github.com/TeamNewPipe/NewPipeExtractor.git
synced 2025-04-26 23:10:34 +05:30
remove searchengine
This commit is contained in:
parent
291de18418
commit
812a785811
@ -1,16 +1,14 @@
|
|||||||
package org.schabi.newpipe.extractor;
|
package org.schabi.newpipe.extractor;
|
||||||
|
|
||||||
import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
|
||||||
import org.schabi.newpipe.extractor.uih.ListUIHandler;
|
import org.schabi.newpipe.extractor.uih.ListUIHandler;
|
||||||
|
|
||||||
import java.util.ArrayList;
|
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
|
||||||
public abstract class ListInfo<T extends InfoItem> extends Info {
|
public abstract class ListInfo<T extends InfoItem> extends Info {
|
||||||
private List<T> relatedItems;
|
private List<T> relatedItems;
|
||||||
private String nextPageUrl = null;
|
private String nextPageUrl = null;
|
||||||
private List<String> contentFilters = new ArrayList<>();
|
private final List<String> contentFilters;
|
||||||
private String sortFilter = "";
|
private final String sortFilter;
|
||||||
|
|
||||||
public ListInfo(int serviceId,
|
public ListInfo(int serviceId,
|
||||||
String id,
|
String id,
|
||||||
@ -24,7 +22,7 @@ public abstract class ListInfo<T extends InfoItem> extends Info {
|
|||||||
this.sortFilter = sortFilter;
|
this.sortFilter = sortFilter;
|
||||||
}
|
}
|
||||||
|
|
||||||
public ListInfo(int serviceId, ListUIHandler listUrlIdHandler, String name) throws ParsingException {
|
public ListInfo(int serviceId, ListUIHandler listUrlIdHandler, String name) {
|
||||||
super(serviceId, listUrlIdHandler, name);
|
super(serviceId, listUrlIdHandler, name);
|
||||||
this.contentFilters = listUrlIdHandler.getContentFilters();
|
this.contentFilters = listUrlIdHandler.getContentFilters();
|
||||||
this.sortFilter = listUrlIdHandler.getSortFilter();
|
this.sortFilter = listUrlIdHandler.getSortFilter();
|
||||||
|
@ -15,7 +15,7 @@ import java.io.IOException;
|
|||||||
|
|
||||||
public class PlaylistInfo extends ListInfo<StreamInfoItem> {
|
public class PlaylistInfo extends ListInfo<StreamInfoItem> {
|
||||||
|
|
||||||
public PlaylistInfo(int serviceId, ListUIHandler urlIdHandler, String name) throws ParsingException {
|
private PlaylistInfo(int serviceId, ListUIHandler urlIdHandler, String name) throws ParsingException {
|
||||||
super(serviceId, urlIdHandler, name);
|
super(serviceId, urlIdHandler, name);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -1,50 +0,0 @@
|
|||||||
package org.schabi.newpipe.extractor.search;
|
|
||||||
|
|
||||||
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
|
|
||||||
|
|
||||||
import java.io.IOException;
|
|
||||||
|
|
||||||
/*
|
|
||||||
* Created by Christian Schabesberger on 10.08.15.
|
|
||||||
*
|
|
||||||
* Copyright (C) Christian Schabesberger 2015 <chris.schabesberger@mailbox.org>
|
|
||||||
* SearchEngine.java is part of NewPipe.
|
|
||||||
*
|
|
||||||
* NewPipe is free software: you can redistribute it and/or modify
|
|
||||||
* it under the terms of the GNU General Public License as published by
|
|
||||||
* the Free Software Foundation, either version 3 of the License, or
|
|
||||||
* (at your option) any later version.
|
|
||||||
*
|
|
||||||
* NewPipe is distributed in the hope that it will be useful,
|
|
||||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
* GNU General Public License for more details.
|
|
||||||
*
|
|
||||||
* You should have received a copy of the GNU General Public License
|
|
||||||
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
|
|
||||||
*/
|
|
||||||
|
|
||||||
public abstract class SearchEngine {
|
|
||||||
public enum Filter {
|
|
||||||
ANY, STREAM, CHANNEL, PLAYLIST
|
|
||||||
}
|
|
||||||
|
|
||||||
public static class NothingFoundException extends ExtractionException {
|
|
||||||
public NothingFoundException(String message) {
|
|
||||||
super(message);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private final InfoItemsSearchCollector collector;
|
|
||||||
|
|
||||||
public SearchEngine(int serviceId) {
|
|
||||||
collector = new InfoItemsSearchCollector(serviceId);
|
|
||||||
}
|
|
||||||
|
|
||||||
protected InfoItemsSearchCollector getInfoItemSearchCollector() {
|
|
||||||
return collector;
|
|
||||||
}
|
|
||||||
|
|
||||||
public abstract InfoItemsSearchCollector search(String query, int page, String contentCountry, Filter filter)
|
|
||||||
throws IOException, ExtractionException;
|
|
||||||
}
|
|
@ -2,21 +2,19 @@ package org.schabi.newpipe.extractor.search;
|
|||||||
|
|
||||||
import org.schabi.newpipe.extractor.InfoItem;
|
import org.schabi.newpipe.extractor.InfoItem;
|
||||||
import org.schabi.newpipe.extractor.ListInfo;
|
import org.schabi.newpipe.extractor.ListInfo;
|
||||||
import org.schabi.newpipe.extractor.uih.ListUIHFactory;
|
|
||||||
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
|
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
|
||||||
import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
||||||
import org.schabi.newpipe.extractor.uih.ListUIHandler;
|
import org.schabi.newpipe.extractor.uih.SearchQIHandler;
|
||||||
|
|
||||||
public class SearchInfo extends ListInfo<InfoItem> {
|
public class SearchInfo extends ListInfo<InfoItem> {
|
||||||
|
|
||||||
private String searchString = "";
|
private String searchString;
|
||||||
private String searchSuggestion = "";
|
private String searchSuggestion;
|
||||||
|
|
||||||
|
|
||||||
public SearchInfo(int serviceId,
|
public SearchInfo(int serviceId,
|
||||||
ListUIHandler urlIdHandler,
|
SearchQIHandler qIHandler,
|
||||||
String searchString) throws ParsingException {
|
String searchString) throws ParsingException {
|
||||||
super(serviceId, urlIdHandler, "Search");
|
super(serviceId, qIHandler, "Search");
|
||||||
this.searchString = searchString;
|
this.searchString = searchString;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -1,71 +0,0 @@
|
|||||||
package org.schabi.newpipe.extractor.search;
|
|
||||||
|
|
||||||
import org.schabi.newpipe.extractor.InfoItem;
|
|
||||||
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
|
|
||||||
|
|
||||||
import javax.annotation.Nonnull;
|
|
||||||
import java.io.IOException;
|
|
||||||
import java.util.ArrayList;
|
|
||||||
import java.util.Collections;
|
|
||||||
import java.util.List;
|
|
||||||
|
|
||||||
/*
|
|
||||||
* Created by Christian Schabesberger on 29.02.16.
|
|
||||||
*
|
|
||||||
* Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
|
|
||||||
* SearchResult.java is part of NewPipe.
|
|
||||||
*
|
|
||||||
* NewPipe is free software: you can redistribute it and/or modify
|
|
||||||
* it under the terms of the GNU General Public License as published by
|
|
||||||
* the Free Software Foundation, either version 3 of the License, or
|
|
||||||
* (at your option) any later version.
|
|
||||||
*
|
|
||||||
* NewPipe is distributed in the hope that it will be useful,
|
|
||||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
* GNU General Public License for more details.
|
|
||||||
*
|
|
||||||
* You should have received a copy of the GNU General Public License
|
|
||||||
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
|
|
||||||
*/
|
|
||||||
|
|
||||||
public class SearchResult {
|
|
||||||
private final int serviceId;
|
|
||||||
public final String suggestion;
|
|
||||||
@Nonnull
|
|
||||||
public final List<InfoItem> resultList;
|
|
||||||
@Nonnull
|
|
||||||
public final List<Throwable> errors;
|
|
||||||
|
|
||||||
public SearchResult(int serviceId, String suggestion, List<InfoItem> results, List<Throwable> errors) {
|
|
||||||
this.serviceId = serviceId;
|
|
||||||
this.suggestion = suggestion;
|
|
||||||
this.resultList = Collections.unmodifiableList(new ArrayList<>(results));
|
|
||||||
this.errors = Collections.unmodifiableList(new ArrayList<>(errors));
|
|
||||||
}
|
|
||||||
|
|
||||||
public static SearchResult getSearchResult(@Nonnull final SearchEngine engine, final String query, final int page,
|
|
||||||
final String languageCode, final SearchEngine.Filter filter)
|
|
||||||
throws IOException, ExtractionException {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
public String getSuggestion() {
|
|
||||||
return suggestion;
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
@Nonnull
|
|
||||||
public List<InfoItem> getResults() {
|
|
||||||
return Collections.unmodifiableList(resultList);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Nonnull
|
|
||||||
public List<Throwable> getErrors() {
|
|
||||||
return errors;
|
|
||||||
}
|
|
||||||
|
|
||||||
public int getServiceId() {
|
|
||||||
return serviceId;
|
|
||||||
}
|
|
||||||
}
|
|
@ -1,84 +0,0 @@
|
|||||||
package org.schabi.newpipe.extractor.services.soundcloud;
|
|
||||||
|
|
||||||
import com.grack.nanojson.JsonArray;
|
|
||||||
import com.grack.nanojson.JsonObject;
|
|
||||||
import com.grack.nanojson.JsonParser;
|
|
||||||
import com.grack.nanojson.JsonParserException;
|
|
||||||
import org.schabi.newpipe.extractor.Downloader;
|
|
||||||
import org.schabi.newpipe.extractor.NewPipe;
|
|
||||||
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
|
|
||||||
import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
|
||||||
import org.schabi.newpipe.extractor.search.InfoItemsSearchCollector;
|
|
||||||
import org.schabi.newpipe.extractor.search.SearchEngine;
|
|
||||||
|
|
||||||
import java.io.IOException;
|
|
||||||
import java.net.URLEncoder;
|
|
||||||
|
|
||||||
public class SoundcloudSearchEngine extends SearchEngine {
|
|
||||||
public static final String CHARSET_UTF_8 = "UTF-8";
|
|
||||||
|
|
||||||
public SoundcloudSearchEngine(int serviceId) {
|
|
||||||
super(serviceId);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public InfoItemsSearchCollector search(String query, int page, String languageCode, Filter filter) throws IOException, ExtractionException {
|
|
||||||
InfoItemsSearchCollector collector = getInfoItemSearchCollector();
|
|
||||||
|
|
||||||
Downloader dl = NewPipe.getDownloader();
|
|
||||||
|
|
||||||
String url = "https://api-v2.soundcloud.com/search";
|
|
||||||
|
|
||||||
switch (filter) {
|
|
||||||
case STREAM:
|
|
||||||
url += "/tracks";
|
|
||||||
break;
|
|
||||||
case CHANNEL:
|
|
||||||
url += "/users";
|
|
||||||
break;
|
|
||||||
case PLAYLIST:
|
|
||||||
url += "/playlists";
|
|
||||||
break;
|
|
||||||
case ANY:
|
|
||||||
// Don't append any parameter to search for everything
|
|
||||||
default:
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
|
|
||||||
url += "?q=" + URLEncoder.encode(query, CHARSET_UTF_8)
|
|
||||||
+ "&client_id=" + SoundcloudParsingHelper.clientId()
|
|
||||||
+ "&limit=10"
|
|
||||||
+ "&offset=" + Integer.toString(page * 10);
|
|
||||||
|
|
||||||
JsonArray searchCollection;
|
|
||||||
try {
|
|
||||||
searchCollection = JsonParser.object().from(dl.download(url)).getArray("collection");
|
|
||||||
} catch (JsonParserException e) {
|
|
||||||
throw new ParsingException("Could not parse json response", e);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (searchCollection.size() == 0) {
|
|
||||||
throw new NothingFoundException("Nothing found");
|
|
||||||
}
|
|
||||||
|
|
||||||
for (Object result : searchCollection) {
|
|
||||||
if (!(result instanceof JsonObject)) continue;
|
|
||||||
//noinspection ConstantConditions
|
|
||||||
JsonObject searchResult = (JsonObject) result;
|
|
||||||
String kind = searchResult.getString("kind", "");
|
|
||||||
switch (kind) {
|
|
||||||
case "user":
|
|
||||||
collector.commit(new SoundcloudChannelInfoItemExtractor(searchResult));
|
|
||||||
break;
|
|
||||||
case "track":
|
|
||||||
collector.commit(new SoundcloudStreamInfoItemExtractor(searchResult));
|
|
||||||
break;
|
|
||||||
case "playlist":
|
|
||||||
collector.commit(new SoundcloudPlaylistInfoItemExtractor(searchResult));
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return collector;
|
|
||||||
}
|
|
||||||
}
|
|
@ -8,7 +8,6 @@ import org.schabi.newpipe.extractor.*;
|
|||||||
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
|
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
|
||||||
import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
||||||
import org.schabi.newpipe.extractor.search.InfoItemsSearchCollector;
|
import org.schabi.newpipe.extractor.search.InfoItemsSearchCollector;
|
||||||
import org.schabi.newpipe.extractor.search.SearchEngine;
|
|
||||||
import org.schabi.newpipe.extractor.search.SearchExtractor;
|
import org.schabi.newpipe.extractor.search.SearchExtractor;
|
||||||
import org.schabi.newpipe.extractor.uih.SearchQIHandler;
|
import org.schabi.newpipe.extractor.uih.SearchQIHandler;
|
||||||
import org.schabi.newpipe.extractor.utils.Parser;
|
import org.schabi.newpipe.extractor.utils.Parser;
|
||||||
@ -70,7 +69,7 @@ public class SoundcloudSearchExtractor extends SearchExtractor {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if (searchCollection.size() == 0) {
|
if (searchCollection.size() == 0) {
|
||||||
throw new SearchEngine.NothingFoundException("Nothing found");
|
throw new SearchExtractor.NothingFoundException("Nothing found");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -1,124 +0,0 @@
|
|||||||
package org.schabi.newpipe.extractor.services.youtube.extractors;
|
|
||||||
|
|
||||||
import org.jsoup.Jsoup;
|
|
||||||
import org.jsoup.nodes.Document;
|
|
||||||
import org.jsoup.nodes.Element;
|
|
||||||
import org.schabi.newpipe.extractor.Downloader;
|
|
||||||
import org.schabi.newpipe.extractor.NewPipe;
|
|
||||||
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
|
|
||||||
import org.schabi.newpipe.extractor.search.InfoItemsSearchCollector;
|
|
||||||
import org.schabi.newpipe.extractor.search.SearchEngine;
|
|
||||||
|
|
||||||
import java.io.IOException;
|
|
||||||
import java.net.URLEncoder;
|
|
||||||
|
|
||||||
|
|
||||||
/*
|
|
||||||
* Created by Christian Schabesberger on 09.08.15.
|
|
||||||
*
|
|
||||||
* Copyright (C) Christian Schabesberger 2015 <chris.schabesberger@mailbox.org>
|
|
||||||
* YoutubeSearchEngine.java is part of NewPipe.
|
|
||||||
*
|
|
||||||
* NewPipe is free software: you can redistribute it and/or modify
|
|
||||||
* it under the terms of the GNU General Public License as published by
|
|
||||||
* the Free Software Foundation, either version 3 of the License, or
|
|
||||||
* (at your option) any later version.
|
|
||||||
*
|
|
||||||
* NewPipe is distributed in the hope that it will be useful,
|
|
||||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
* GNU General Public License for more details.
|
|
||||||
*
|
|
||||||
* You should have received a copy of the GNU General Public License
|
|
||||||
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
|
|
||||||
*/
|
|
||||||
|
|
||||||
public class YoutubeSearchEngine extends SearchEngine {
|
|
||||||
|
|
||||||
private static final String TAG = YoutubeSearchEngine.class.toString();
|
|
||||||
public static final String CHARSET_UTF_8 = "UTF-8";
|
|
||||||
|
|
||||||
public YoutubeSearchEngine(int serviceId) {
|
|
||||||
super(serviceId);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public InfoItemsSearchCollector search(String query, int page, String languageCode, Filter filter)
|
|
||||||
throws IOException, ExtractionException {
|
|
||||||
InfoItemsSearchCollector collector = getInfoItemSearchCollector();
|
|
||||||
Downloader downloader = NewPipe.getDownloader();
|
|
||||||
|
|
||||||
String url = "https://www.youtube.com/results"
|
|
||||||
+ "?q=" + URLEncoder.encode(query, CHARSET_UTF_8)
|
|
||||||
+ "&page=" + Integer.toString(page + 1);
|
|
||||||
|
|
||||||
switch (filter) {
|
|
||||||
case STREAM:
|
|
||||||
url += "&sp=EgIQAVAU";
|
|
||||||
break;
|
|
||||||
case CHANNEL:
|
|
||||||
url += "&sp=EgIQAlAU"; //EgIQA( lowercase L )AU
|
|
||||||
break;
|
|
||||||
case PLAYLIST:
|
|
||||||
url += "&sp=EgIQA1AU"; //EgIQA( one )AU
|
|
||||||
break;
|
|
||||||
case ANY:
|
|
||||||
// Don't append any parameter to search for everything
|
|
||||||
default:
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
|
|
||||||
String site;
|
|
||||||
//String url = builder.build().toString();
|
|
||||||
//if we've been passed a valid language code, append it to the URL
|
|
||||||
if (!languageCode.isEmpty()) {
|
|
||||||
//assert Pattern.matches("[a-z]{2}(-([A-Z]{2}|[0-9]{1,3}))?", languageCode);
|
|
||||||
site = downloader.download(url, languageCode);
|
|
||||||
} else {
|
|
||||||
site = downloader.download(url);
|
|
||||||
}
|
|
||||||
|
|
||||||
Document doc = Jsoup.parse(site, url);
|
|
||||||
Element list = doc.select("ol[class=\"item-section\"]").first();
|
|
||||||
|
|
||||||
for (Element item : list.children()) {
|
|
||||||
/* First we need to determine which kind of item we are working with.
|
|
||||||
Youtube depicts five different kinds of items on its search result page. These are
|
|
||||||
regular videos, playlists, channels, two types of video suggestions, and a "no video
|
|
||||||
found" item. Since we only want videos, we need to filter out all the others.
|
|
||||||
An example for this can be seen here:
|
|
||||||
https://www.youtube.com/results?search_query=asdf&page=1
|
|
||||||
|
|
||||||
We already applied a filter to the url, so we don't need to care about channels and
|
|
||||||
playlists now.
|
|
||||||
*/
|
|
||||||
|
|
||||||
Element el;
|
|
||||||
|
|
||||||
// both types of spell correction item
|
|
||||||
if ((el = item.select("div[class*=\"spell-correction\"]").first()) != null) {
|
|
||||||
if (list.children().size() == 1) {
|
|
||||||
throw new NothingFoundException("Did you mean: " + el.select("a").first().text());
|
|
||||||
}
|
|
||||||
// search message item
|
|
||||||
} else if ((el = item.select("div[class*=\"search-message\"]").first()) != null) {
|
|
||||||
throw new NothingFoundException(el.text());
|
|
||||||
|
|
||||||
// video item type
|
|
||||||
} else if ((el = item.select("div[class*=\"yt-lockup-video\"]").first()) != null) {
|
|
||||||
collector.commit(new YoutubeStreamInfoItemExtractor(el));
|
|
||||||
} else if ((el = item.select("div[class*=\"yt-lockup-channel\"]").first()) != null) {
|
|
||||||
collector.commit(new YoutubeChannelInfoItemExtractor(el));
|
|
||||||
} else if ((el = item.select("div[class*=\"yt-lockup-playlist\"]").first()) != null &&
|
|
||||||
item.select(".yt-pl-icon-mix").isEmpty()) {
|
|
||||||
collector.commit(new YoutubePlaylistInfoItemExtractor(el));
|
|
||||||
} else {
|
|
||||||
// noinspection ConstantConditions
|
|
||||||
// simply ignore not known items
|
|
||||||
// throw new ExtractionException("unexpected element found: \"" + item + "\"");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return collector;
|
|
||||||
}
|
|
||||||
}
|
|
Loading…
x
Reference in New Issue
Block a user