New version / CaptchaBypass testing

app/routes.py

@@ -1,4 +1,3 @@
-
 import datetime
 import glob
 import json
@@ -30,8 +29,8 @@ from app.forms import LoginForm, RegistrationForm, EmptyForm, SearchForm, Channe
 from app.models import User, twitterPost, ytPost, Post, youtubeFollow, twitterFollow
 from youtube import comments, utils, channel as ytch, search as yts
 from youtube import watch as ytwatch
+
 #########################################
-from youtube_data import search as yts
 
 #########################################
 
@@ -326,6 +325,10 @@ def ytsearch():
         else:
             prev_page = "/ytsearch?q={q}&s={s}&p={p}".format(q=query, s=sort, p=int(page) - 1)
 
+        for video in results['videos']:
+            hostname = urllib.parse.urlparse(video['videoThumb']).netloc
+            video['videoThumb'] = video['videoThumb'].replace("https://{}".format(hostname), "") + "&host=" + hostname
+
         for channel in results['channels']:
             if config['nginxVideoStream']:
                 channel['thumbnail'] = channel['thumbnail'].replace("~", "/")
@@ -342,8 +345,6 @@ def ytsearch():
 @app.route('/ytfollow/<channelId>', methods=['POST'])
 @login_required
 def ytfollow(channelId):
-    form = EmptyForm()
-    if form.validate_on_submit():
     r = followYoutubeChannel(channelId)
     return redirect(request.referrer)
 
@@ -376,8 +377,6 @@ def followYoutubeChannel(channelId):
 @app.route('/ytunfollow/<channelId>', methods=['POST'])
 @login_required
 def ytunfollow(channelId):
-    form = EmptyForm()
-    if form.validate_on_submit():
     unfollowYoutubeChannel(channelId)
     return redirect(request.referrer)
 
@@ -404,27 +403,38 @@ def unfollowYoutubeChannel(channelId):
 def channel(id):
     form = ChannelForm()
     button_form = EmptyForm()
-    data = requests.get('https://www.youtube.com/feeds/videos.xml?channel_id={id}'.format(id=id))
-    data = feedparser.parse(data.content)
 
-    channelData = YoutubeSearch.channelInfo(id)
+    page = request.args.get('p', None)
+    sort = request.args.get('s', None)
+    if page is None:
+        page = 1
+    if sort is None:
+        sort = 3
 
-    for video in channelData[1]:
+    data = ytch.get_channel_tab_info(id, page, sort)
+
+    for video in data['items']:
         if config['nginxVideoStream']:
-            hostName = urllib.parse.urlparse(video['videoThumb']).netloc
+            hostName = urllib.parse.urlparse(video['thumbnail'][1:]).netloc
-            video['videoThumb'] = video['videoThumb'].replace("https://{}".format(hostName), "").replace("hqdefault",
+            video['thumbnail'] = video['thumbnail'].replace("https://{}".format(hostName), "")[1:].replace("hqdefault",
                                                                                                        "mqdefault") + "&host=" + hostName
         else:
-            video['videoThumb'] = video['videoThumb'].replace('/', '~')
+            video['thumbnail'] = video['thumbnail'].replace('/', '~')
-    if config['nginxVideoStream']:
-        hostName = urllib.parse.urlparse(channelData[0]['avatar']).netloc
-        channelData[0]['avatar'] = channelData[0]['avatar'].replace("https://{}".format(hostName),
-                                                                    "") + "?host=" + hostName
-    else:
-        channelData[0]['avatar'] = channelData[0]['avatar'].replace('/', '~')
 
-    return render_template('channel.html', form=form, btform=button_form, channel=channelData[0], videos=channelData[1],
+    if config['nginxVideoStream']:
-                           restricted=config['restrictPublicUsage'], config=config)
+        hostName = urllib.parse.urlparse(data['avatar'][1:]).netloc
+        data['avatar'] = data['avatar'].replace("https://{}".format(hostName), "")[1:] + "?host=" + hostName
+    else:
+        data['avatar'] = data['avatar'].replace('/', '~')
+
+    next_page = "/channel/{q}?s={s}&p={p}".format(q=id, s=sort, p=int(page) + 1)
+    if int(page) == 1:
+        prev_page = "/channel/{q}?s={s}&p={p}".format(q=id, s=sort, p=1)
+    else:
+        prev_page = "/channel/{q}?s={s}&p={p}".format(q=id, s=sort, p=int(page) - 1)
+
+    return render_template('channel.html', form=form, btform=button_form, data=data,
+                           restricted=config['restrictPublicUsage'], config=config, next_page=next_page, prev_page=prev_page)
 
 
 def get_best_urls(urls):
@@ -454,18 +464,13 @@ def get_live_urls(urls):
 def watch():
     id = request.args.get('v', None)
     info = ytwatch.extract_info(id, False, playlist_id=None, index=None)
-<<<<<<< Updated upstream
-<<<<<<< Updated upstream
-<<<<<<< Updated upstream
-    # Use nginx
-    best_formats = ["22", "18", "34", "35", "36", "37", "38", "43", "44", "45", "46"]
-=======
-=======
->>>>>>> Stashed changes
-=======
->>>>>>> Stashed changes
-    vsources = ytwatch.get_video_sources(info, False)
 
+    retry = 3
+    while retry != 0 and info['playability_error'] == 'Could not find player':
+        info=ytwatch.extract_info(id, False, playlist_id=None, index=None)
+        retry -= 1
+
+    vsources = ytwatch.get_video_sources(info, False)
     # Retry 3 times if no sources are available.
     retry = 3
     while retry != 0 and len(vsources) == 0:
@@ -477,26 +482,29 @@ def watch():
         source['src'] = source['src'].replace("https://{}".format(hostName), "") + "&host=" + hostName
 
     # Parse video formats
->>>>>>> Stashed changes
     for v_format in info['formats']:
         hostName = urllib.parse.urlparse(v_format['url']).netloc
         v_format['url'] = v_format['url'].replace("https://{}".format(hostName), "") + "&host=" + hostName
-        if v_format['audio_bitrate'] is not None and v_format['vcodec'] is not None:
+        if v_format['audio_bitrate'] is not None and v_format['vcodec'] is None:
-            v_format['video_valid'] = True
-        elif v_format['audio_bitrate'] is not None and v_format['vcodec'] is None:
             v_format['audio_valid'] = True
 
+    # Markup description
+    try:
         info['description'] = Markup(bleach.linkify(info['description'].replace("\n", "<br>")))
+    except AttributeError or TypeError:
+        print(info['description'])
+
 
     # Get comments
     videocomments = comments.video_comments(id, sort=0, offset=0, lc='', secret_key='')
     videocomments = utils.post_process_comments_info(videocomments)
-
     if videocomments is not None:
         videocomments.sort(key=lambda x: x['likes'], reverse=True)
 
-    info['rating'] = str((info['like_count']/(info['like_count']+info['dislike_count']))*100)[0:4]
+    # Calculate rating %
-    return render_template("video.html", info=info, title='{}'.format(info['title']), config=config, videocomments=videocomments)
+    info['rating'] = str((info['like_count'] / (info['like_count'] + info['dislike_count'])) * 100)[0:4]
+    return render_template("video.html", info=info, title='{}'.format(info['title']), config=config,
+                           videocomments=videocomments, vsources=vsources)
 
 
 def markupString(string):
@@ -745,15 +753,17 @@ def register():
     return render_template('register.html', title='Register', registrations=REGISTRATIONS, form=form, config=config)
 
 
-@app.route('/registrations_status/icon')
+@app.route('/status')
-def registrations_status_icon():
+def status():
     count = db.session.query(User).count()
     if count >= config['maxInstanceUsers'] or config['maxInstanceUsers'] == 0:
-        return redirect(url_for('static', filename='img/close.png'))
+        filen = url_for('static', filename='img/close.png')
+        caniregister = False
     else:
         filen = url_for('static', filename='img/open.png')
         caniregister = True
 
+    return render_template('status.html', title='STATUS', count=count, max=config['maxInstanceUsers'], file=filen, cani=caniregister)
 
 @app.route('/error/<errno>')
 def error(errno):

app/templates/_video_item.html

@@ -1,46 +1,35 @@
-<div class="card">
+<div class="ui card">
-    <div class="image">
+  <a class="image" href="{{url_for('watch', v=video.id, _method='GET')}}">
-        {%if config.nginxVideoStream%}
+    <img src="https://yotter.xyz{{video.videoThumb}}">
-        <img alt="Thumbnail" src="{{video.videoThumb}}">
+  </a>
-        {%else%}
-        <img alt="Thumbnail" src="/img/{{video.videoThumb.replace('/', '~')}}">
-        {%endif%}
-    </div>
   <div class="content">
-        {% if video.views == "Livestream" %}
+    <a class="header" href="{{url_for('watch', v=video.id, _method='GET')}}">{{video.videoTitle}}</a>
-        <a class="video-title break-word" href="#">{{video.videoTitle}}</a>
-        {% else %}
-        <a class="video-title break-word" href="{{url_for('watch', v=video.id, _method='GET')}}">{{video.videoTitle}}</a>
-        {% endif %}
     <div class="meta">
       <a class="break-word" href="{{url_for('channel', id=video.channelId)}}">{{video.channelName}}</a>
     </div>
-        <div class="description break-word">
-            {{video.description}}
-        </div>
   </div>
     <div class="extra content">
         {% if video.isLive == "Livestream" or video.isLive %}
-            <span class="right floated">
+            <span class="left floated like">
                 <i class="red circle icon"></i>
-                ‎‎{{video.views}}
+                {{video.views}}
             </span>
         {% else %}
-            <span class="right floated">
+            <span class="left floated like">
                 <i class="eye icon"></i>
-                ‎‎{{video.views}}
+                {{video.views}}
             </span>
         {% endif %}
 
         {% if video.timeStamp == "Scheduled" or video.isUpcoming %}
-            <span class="right floated">
+            <span class="right floated star">
                 <i class="blue clock icon"></i>
-                {{video.timeStamp}}‎‎‎‎            ‎‎‎
+                {{video.timeStamp}}
             </span>
         {% else %}
-            <span class="right floated">
+            <span class="right floated star">
                 <i class="clock icon"></i>
-                {{video.timeStamp}}‎‎‎‎            ‎‎‎
+                {{video.timeStamp}}
             </span>
         {% endif %}
         <span>

app/templates/channel.html

@@ -1,58 +1,94 @@
 {% extends "base.html" %}
 
+
 {% block content %}
-    <div class="blue ui centered card">
+<div class="ui center aligned text container">
-        <div class="content">
+    <div class="ui centered vertical segment">
-            <div class="center aligned author">
+        <h2 class="ui header">
-                {%if config.nginxVideoStream%}
+          <img src="{{data.avatar}}" class="ui circular image">
-                    <img alt="Thumbnail" src="{{channel.avatar}}">
+          {{data.channel_name}}
-                {%else%}
+        </h2>
-                    <img alt="Thumbnail" src="/img/{{channel.avatar.replace('/', '~')}}">
-                {%endif%}
     </div>
-            <div class="center aligned header"><a href="">{{channel.name}}</a></div>
+    <div class="ui vertical segment">
-            <div class="center aligned description">
+        <p>{{data.short_description}}</p>
-                    <div class="statistic">
+    </div>
+    <div class="ui vertical segment">
+          <div class="ui tiny statistic">
             <div class="value">
-                            <i class="users icon"></i>{{channel.subCount}}
+                {%if data.approx_suscriber_count == None%}
+                    <i class="user icon"></i> ?
+                {%else%}
+                    <i class="user icon"></i> {{data.approx_subscriber_count}}
+                {%endif%}
             </div>
             <div class="label">
                Followers
             </div>
           </div>
-            </div>
+{% if restricted or current_user.is_authenticated %}
-        </div>
+    {% if not current_user.is_following_yt(data.channel_id) %}
-        {% if restricted or current_user.is_authenticated %}
+        <form action="{{ url_for('ytfollow', channelId=data.channel_id) }}" method="post">
-        <div class="center aligned extra content">
+            <button type="submit" value="Submit" class="ui red button">
-            {% if not current_user.is_following_yt(channel.id) %}
+                <i class="user icon"></i>
-                <p>
+                Suscribe
-                    <form action="{{ url_for('ytfollow', channelId=channel.id) }}" method="post">
+            </button>
-                        {{ btform.hidden_tag() }}
-                        {{ btform.submit(value='Follow') }}
         </form>
-                </p>
     {% else %}
-                <p>
+        <form action="{{ url_for('ytunfollow', channelId=data.channel_id) }}" method="post">
-                    <form action="{{ url_for('ytunfollow', channelId=channel.id) }}" method="post">
+            <button type="submit" value="Submit" class="ui red active button">
-                        {{ btform.hidden_tag() }}
+                <i class="user icon"></i>
-                        {{ btform.submit(value='Unfollow') }}
+                Unsuscribe
+            </button>
         </form>
-                </p>
+    {%endif%}
-            {% endif %}           
+{%endif%}
-        </div>
-        {% endif %}
-        </div>
     </div>
+</div>
 
     <br>
     <br> 
-    {% if not videos %}
+    {% if data['error'] != None %}
         {% include '_empty_feed.html' %}
     {% else %}
     <div class="ui centered cards">
-        {% for video in videos %}
+        {% for video in data['items'] %}
-            {% include '_video_item.html' %}
+            <div class="ui card">
+              <a class="image" href="{{url_for('watch', v=video.id, _method='GET')}}">
+                <img src="https://yotter.xyz{{video.thumbnail}}">
+              </a>
+              <div class="content">
+                <a class="header" href="{{url_for('watch', v=video.id, _method='GET')}}">{{video.title}}</a>
+                <div class="meta">
+                  <a class="break-word" href="{{url_for('channel', id=video.channel_id)}}">{{data.channel_name}}</a>
+                </div>
+              </div>
+                <div class="extra content">
+                    <span class="left floated like">
+                        <i class="eye icon"></i>
+                        {{video.approx_view_count}}
+                    </span>
+
+                    {%if video.duration == "PREMIERING NOW" or video.duration == "LIVE"%}
+                        <span class="right floated star">
+                            <i class="red circle icon"></i>
+                            LIVE
+                        </span>
+                    {%else%}
+                        <span class="right floated star">
+                            <i class="clock icon"></i>
+                            {{video.time_published}}
+                        </span>
+                    {%endif%}
+                </div>
+            </div>
         {% endfor %}
     </div>
     {% endif %}
+
+    <br>
+    <div class="ui center aligned text container">
+        <a href="{{prev_page}}"> <button class="ui left attached button"><i class="angle red left icon"></i></button> </a>
+        <a href="{{next_page}}"> <button class="right attached ui button"><i class="angle red right icon"></i></button></a>
+    </div>
+    <br>
 {% endblock %}

app/templates/status.html

@@ -0,0 +1,46 @@
+{% extends "base.html" %}
+{% block content %}
+<div class="ui text container center aligned centered">
+    <div class="ui placeholder segment">
+      <div class="ui two column stackable center aligned grid">
+        <div class="ui vertical divider">
+          {%if cani%}
+            :)
+          {%else%}
+            :(
+          {%endif%}
+        </div>
+        <div class="middle aligned row">
+          <div class="column">
+            <h3 class="ui header"> Capacity </h3>
+            <div class="ui icon header">
+              {%if cani%}
+                <i class="green users icon"></i>
+              {%else%}
+                <i class="red users icon"></i>
+              {%endif%}
+              {{count}}/{{max}}
+            </div>
+          </div>
+          <div class="column">
+            <div class="ui icon header">
+              <i class="user circle outline icon"></i>
+              Can I register?
+            </div>
+
+              {%if cani%}
+                <a href="/register"><div class="ui green button">
+                  Yes!
+                </div></a>
+              {%else%}
+                <a href="#!"><div class="ui disabled red button">
+                  It's full!
+                </div></a>
+              {%endif%}
+
+          </div>
+        </div>
+      </div>
+    </div>
+</div>
+{%endblock%}

app/templates/video.html

@@ -34,20 +34,18 @@
         </div>
     {%else%}
         <div class="video-js-responsive-container vjs-hd">
-            <video class="video-js vjs-default-skin" 
+            <video-js autofocus class="video-js vjs-default-skin"
             data-setup='{ "playbackRates": [0.5, 0.75, 1, 1.25,1.5, 1.75, 2] }'
             width="1080"
             controls
             buffered
             preload="none">
             {% if config.nginxVideoStream %}
-                {% for format in info.formats %}
+                {% for source in vsources %}
-                    {% if format.video_valid %}
+                        <source src="{{source.src}}" type="{{source.type}}">
-                        <source src="{{format.url}}" type="video/{{format.ext}}">
-                    {% endif %}
                 {% endfor %}
             {% endif %}
-            </video>
+            </video-js>
         </div>
     {%endif%}
 
@@ -99,7 +97,6 @@
             
 <script src="{{ url_for('static',filename='video.min.js') }}"></script>
 {% if info.live %}
-    <p>Active</p>
     <script src="{{ url_for('static',filename='videojs-http-streaming.min.js')}}"></script>
     <script>
         var player = videojs('live');

youtube/channel.py

@@ -1,20 +1,16 @@
 import base64
-from youtube import util, yt_data_extract, local_playlist, subscriptions
-from youtube import yt_app
-
-import urllib
 import json
-from string import Template
-import youtube.proto as proto
-import html
 import math
-import gevent
 import re
-import cachetools.func
 import traceback
+import urllib
 
+import cachetools.func
 import flask
-from flask import request
+import gevent
+
+import youtube.proto as proto
+from youtube import util, yt_data_extract
 
 headers_desktop = (
     ('Accept', '*/*'),
@@ -109,7 +105,7 @@ def channel_ctoken_v1(channel_id, page, sort, tab, view=1):
 
     return base64.urlsafe_b64encode(pointless_nest).decode('ascii')
 
-def get_channel_tab(channel_id, page="1", sort=3, tab='videos', view=1, print_status=True):
+def get_channel_tab_info(channel_id, page="1", sort=3, tab='videos', view=1, print_status=True):
     message = 'Got channel tab' if print_status else None
 
     if int(sort) == 2 and int(page) > 1:
@@ -128,7 +124,11 @@ def get_channel_tab(channel_id, page="1", sort=3, tab='videos', view=1, print_st
             headers_desktop + generic_cookie,
             debug_name='channel_tab', report_text=message)
 
-    return content
+    info = yt_data_extract.extract_channel_info(json.loads(content), tab)
+    if info['error'] is not None:
+        return False
+    post_process_channel_info(info)
+    return info
 
 # cache entries expire after 30 minutes
 @cachetools.func.ttl_cache(maxsize=128, ttl=30*60)
@@ -259,23 +259,4 @@ def get_channel_page_general_url(base_url, tab, request, channel_id=None):
         **info
     )
 
-@yt_app.route('/channel/<channel_id>/')
-@yt_app.route('/channel/<channel_id>/<tab>')
-def get_channel_page(channel_id, tab='videos'):
-    return get_channel_page_general_url('https://www.youtube.com/channel/' + channel_id, tab, request, channel_id)
-
-@yt_app.route('/user/<username>/')
-@yt_app.route('/user/<username>/<tab>')
-def get_user_page(username, tab='videos'):
-    return get_channel_page_general_url('https://www.youtube.com/user/' + username, tab, request)
-
-@yt_app.route('/c/<custom>/')
-@yt_app.route('/c/<custom>/<tab>')
-def get_custom_c_page(custom, tab='videos'):
-    return get_channel_page_general_url('https://www.youtube.com/c/' + custom, tab, request)
-
-@yt_app.route('/<custom>')
-@yt_app.route('/<custom>/<tab>')
-def get_toplevel_custom_page(custom, tab='videos'):
-    return get_channel_page_general_url('https://www.youtube.com/' + custom, tab, request)
 

youtube/channels.py

@@ -0,0 +1,213 @@
+from youtube import proto
+from flask import Markup as mk
+import requests
+import base64
+import json
+import re
+
+# From: https://github.com/user234683/youtube-local/blob/master/youtube/channel.py
+# SORT:
+# videos:
+#    Popular - 1
+#    Oldest - 2
+#    Newest - 3
+# playlists:
+#    Oldest - 2
+#    Newest - 3
+#    Last video added - 4
+
+# view:
+# grid: 0 or 1
+# list: 2
+
+headers = {
+        'Host': 'www.youtube.com',
+        'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64)',
+        'Accept': '*/*',
+        'Accept-Language': 'en-US,en;q=0.5',
+        'X-YouTube-Client-Name': '1',
+        'X-YouTube-Client-Version': '2.20180418',
+    }
+real_cookie = (('Cookie', 'VISITOR_INFO1_LIVE=8XihrAcN1l4'),)
+generic_cookie = (('Cookie', 'VISITOR_INFO1_LIVE=ST1Ti53r4fU'),)
+
+
+def channel_ctoken_desktop(channel_id, page, sort, tab, view=1):
+    # see https://github.com/iv-org/invidious/issues/1319#issuecomment-671732646
+    # page > 1 doesn't work when sorting by oldest
+    offset = 30*(int(page) - 1)
+    schema_number = {
+        3: 6307666885028338688,
+        2: 17254859483345278706,
+        1: 16570086088270825023,
+    }[int(sort)]
+    page_token = proto.string(61, proto.unpadded_b64encode(proto.string(1,
+            proto.uint(1, schema_number) + proto.string(2,
+                proto.string(1, proto.unpadded_b64encode(proto.uint(1,offset)))
+            )
+    )))
+
+    tab = proto.string(2, tab )
+    sort = proto.uint(3, int(sort))
+    #page = proto.string(15, str(page) )
+
+    shelf_view = proto.uint(4, 0)
+    view = proto.uint(6, int(view))
+    continuation_info = proto.string(3,
+        proto.percent_b64encode(tab + sort + shelf_view + view + page_token)
+    )
+
+    channel_id = proto.string(2, channel_id )
+    pointless_nest = proto.string(80226972, channel_id + continuation_info)
+
+    return base64.urlsafe_b64encode(pointless_nest).decode('ascii')
+
+def channel_ctoken_mobile(channel_id, page, sort, tab, view=1):
+    tab = proto.string(2, tab )
+    sort = proto.uint(3, int(sort))
+    page = proto.string(15, str(page) )
+    # example with shelves in videos tab: https://www.youtube.com/channel/UCNL1ZadSjHpjm4q9j2sVtOA/videos
+    shelf_view = proto.uint(4, 0)
+    view = proto.uint(6, int(view))
+    continuation_info = proto.string( 3, proto.percent_b64encode(tab + view + sort + shelf_view + page) )
+
+    channel_id = proto.string(2, channel_id )
+    pointless_nest = proto.string(80226972, channel_id + continuation_info)
+
+    return base64.urlsafe_b64encode(pointless_nest).decode('ascii')
+
+
+def id_or_username(string):
+    cidRegex = "^UC.{22}$"
+    if re.match(cidRegex, string):
+        return "channel"
+    else:
+        return "user"
+
+def get_channel_videos_tab(content):
+    tabs = content['contents']['twoColumnBrowseResultsRenderer']['tabs']
+    for tab in tabs:
+        if tab['title'] != "Videos":
+            continue
+        else:
+            return tab
+
+def get_video_items_from_tab(tab):
+    items = []
+    for item in tab:
+        try:
+            if item['gridVideoRenderer']:
+                items.append(item)
+            else:
+                continue
+        except KeyError:
+            continue
+    return items
+
+def get_info_grid_video_item(item, channel=None):
+    item = item['gridVideoRenderer']
+    thumbnailOverlays = item['thumbnailOverlays']
+    published = ""
+    views = ""
+    isLive = False
+    isUpcoming = False
+    try:
+        if 'UPCOMING' in str(thumbnailOverlays):
+            start_time = item['upcomingEventData']['startTime']
+            isUpcoming = True
+            views = "-"
+            published = "Scheduled"
+    except KeyError:
+        isUpcoming = False
+
+    try:
+        if 'LIVE' in str(thumbnailOverlays):
+            isLive = True
+            try:
+                views = item['viewCountText']['simpleText']
+            except:
+                views = "Live"
+            try:
+                duration = item['lengthText']['simpleText']
+            except:
+                duration = "-"
+            if published != "Scheduled":
+                try:
+                    published = item['publishedTimeText']['simpleText']
+                except KeyError:
+                    published = "None"
+    except KeyError:
+        isUpcoming = False
+        isLive = False
+
+    if not isUpcoming and not isLive:
+        views = item['viewCountText']['simpleText']
+        published = item['publishedTimeText']['simpleText']
+        try:
+            duration = item['lengthText']['simpleText']
+        except:
+            duration = "?"
+
+    video = {
+        'videoTitle':item['title']['runs'][0]['text'],
+        'description':"",
+        'views':views,
+        'timeStamp':published,
+        'duration':duration,
+        'channelName':channel['username'],
+        'authorUrl':"/channel/{}".format(channel['channelId']),
+        'channelId':channel['channelId'],
+        'id':item['videoId'],
+        'videoUrl':"/watch?v={}".format(item['videoId']),
+        'isLive':isLive,
+        'isUpcoming':isUpcoming,
+        'videoThumb':item['thumbnail']['thumbnails'][0]['url']
+    }
+    return video
+
+def get_author_info_from_channel(content):
+    hmd = content['metadata']['channelMetadataRenderer']
+    cmd = content['header']['c4TabbedHeaderRenderer']
+    description = mk(hmd['description'])
+    channel = {
+        "channelId": cmd['channelId'],
+        "username": cmd['title'],
+        "thumbnail": "https:{}".format(cmd['avatar']['thumbnails'][0]['url'].replace("/", "~")),
+        "description":description,
+        "suscribers": cmd['subscriberCountText']['runs'][0]['text'].split(" ")[0],
+        "banner": cmd['banner']['thumbnails'][0]['url']
+    }
+    return channel
+
+def get_channel_info(channelId, videos=True, page=1, sort=3):
+    if id_or_username(channelId) == "channel":
+        videos = []
+        ciUrl = "https://www.youtube.com/channel/{}".format(channelId)
+        mainUrl = "https://www.youtube.com/browse_ajax?ctoken={}".format(channel_ctoken_desktop(channelId, page, sort, "videos"))
+        content = json.loads(requests.get(mainUrl, headers=headers).text)
+        req = requests.get(ciUrl, headers=headers).text
+
+        start = (
+            req.index('window["ytInitialData"]')
+            + len('window["ytInitialData"]')
+            + 3
+        )
+
+        end = req.index("};", start) + 1
+        jsonIni = req[start:end]
+        data = json.loads(jsonIni)
+
+        #videosTab = get_channel_videos_tab(content)
+        authorInfo = get_author_info_from_channel(data)
+        if videos:
+            gridVideoItemList = get_video_items_from_tab(content[1]['response']['continuationContents']['gridContinuation']['items'])
+            for video in gridVideoItemList:
+                vid = get_info_grid_video_item(video, authorInfo)
+                videos.append(vid)
+            print({"channel":authorInfo, "videos":videos})
+            return {"channel":authorInfo, "videos":videos}
+        else:
+            return {"channel":authorInfo}
+
+    else:
+        baseUrl = "https://www.youtube.com/user/{}".format(channelId)

youtube/search.py

@@ -1,38 +1,10 @@
+from youtube import proto
+from youtube import utils
+from flask import Markup
+import urllib.parse
+import requests
 import base64
 import json
-import urllib
-
-import flask
-from flask import request
-from werkzeug.exceptions import abort
-
-from youtube import util, yt_data_extract, proto
-from youtube import yt_app
-
-# Sort: 1
-    # Upload date: 2
-    # View count: 3
-    # Rating: 1
-    # Relevance: 0
-# Offset: 9
-# Filters: 2
-    # Upload date: 1
-    # Type: 2
-    # Duration: 3
-
-
-features = {
-    '4k': 14,
-    'hd': 4,
-    'hdr': 25,
-    'subtitles': 5,
-    'creative_commons': 6,
-    '3d': 7,
-    'live': 8,
-    'purchased': 9,
-    '360': 15,
-    'location': 23,
-}
 
 def page_number_to_sp_parameter(page, autocorrect, sort, filters):
     offset = (int(page) - 1)*20    # 20 results per page
@@ -41,8 +13,8 @@ def page_number_to_sp_parameter(page, autocorrect, sort, filters):
     result = proto.uint(1, sort) + filters_enc + autocorrect + proto.uint(9, offset) + proto.string(61, b'')
     return base64.urlsafe_b64encode(result).decode('ascii')
 
-def get_search_json(query, page, autocorrect, sort, filters):
+def search_by_terms(search_terms, page, autocorrect, sort, filters):
-    url = "https://www.youtube.com/results?search_query=" + urllib.parse.quote_plus(query)
+    url = "https://www.youtube.com/results?search_query=" + urllib.parse.quote_plus(search_terms)
     headers = {
         'Host': 'www.youtube.com',
         'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64)',
@@ -52,54 +24,145 @@ def get_search_json(query, page, autocorrect, sort, filters):
         'X-YouTube-Client-Version': '2.20180418',
     }
     url += "&pbj=1&sp=" + page_number_to_sp_parameter(page, autocorrect, sort, filters).replace("=", "%3D")
-    content = util.fetch_url(url, headers=headers, report_text="Got search results", debug_name='search_results')
+    content = requests.get(url, headers=headers).text
     info = json.loads(content)
-    return info
+    videos = get_videos_from_search(info)
+    channels = get_channels_from_search(info)
 
+    results = {
+        "videos": videos,
+        "channels": channels
+    }
+    return results
 
-@yt_app.route('/search')
+def get_channels_from_search(search):
-def get_search_page():
+    results = []
-    if len(request.args) == 0:
+    search = search[1]['response']
-        return flask.render_template('base.html', title="Search")
+    primaryContents = search['contents']['twoColumnSearchResultsRenderer']['primaryContents']
+    contents = primaryContents['sectionListRenderer']['contents']
 
-    if 'query' not in request.args:
+    for content in contents:
-        abort(400)
+        try:
+            items = content['itemSectionRenderer']['contents']
+        except:
+            continue
 
-    query = request.args.get("query")
+    for item in items:
-    page = request.args.get("page", "1")
+        try:
-    autocorrect = int(request.args.get("autocorrect", "1"))
+            item['channelRenderer']
-    sort = int(request.args.get("sort", "0"))
+            channel = get_channel_renderer_item_info(item['channelRenderer'])
-    filters = {}
+            results.append(channel)
-    filters['time'] = int(request.args.get("time", "0"))
+        except KeyError:
-    filters['type'] = int(request.args.get("type", "0"))
+            continue
-    filters['duration'] = int(request.args.get("duration", "0"))
+    return results
-    polymer_json = get_search_json(query, page, autocorrect, sort, filters)
 
-    search_info = yt_data_extract.extract_search_info(polymer_json)
+def get_channel_renderer_item_info(item):
-    if search_info['error']:
+    try:
-        return flask.render_template('error.html', error_message = search_info['error'])
+        suscribers = item['subscriberCountText']['simpleText'].split(" ")[0]
+    except:
+        suscribers = "?"
     
-    for extract_item_info in search_info['items']:
+    try:
-        util.prefix_urls(extract_item_info)
+        description = utils.get_description_snippet_text(item['descriptionSnippet']['runs'])
-        util.add_extra_html_info(extract_item_info)
+    except KeyError:
+        description = ""
 
-    corrections = search_info['corrections']
+    try:
-    if corrections['type'] == 'did_you_mean':
+        channel = {
-        corrected_query_string = request.args.to_dict(flat=False)
+            "channelId": item['channelId'],
-        corrected_query_string['query'] = [corrections['corrected_query']]
+            "username": item['title']['simpleText'],
-        corrections['corrected_query_url'] = util.URL_ORIGIN + '/search?' + urllib.parse.urlencode(corrected_query_string, doseq=True)
+            "thumbnail": "https:{}".format(item['thumbnail']['thumbnails'][0]['url'].replace("/", "~")),
-    elif corrections['type'] == 'showing_results_for':
+            "description": Markup(str(description)),
-        no_autocorrect_query_string = request.args.to_dict(flat=False)
+            "suscribers": suscribers,
-        no_autocorrect_query_string['autocorrect'] = ['0']
+            "videos": item['videoCountText']['runs'][0]['text']
-        no_autocorrect_query_url = util.URL_ORIGIN + '/search?' + urllib.parse.urlencode(no_autocorrect_query_string, doseq=True)
+        }
-        corrections['original_query_url'] = no_autocorrect_query_url
+    except KeyError:
+        channel = {
+            "channelId": item['channelId'],
+            "username": item['title']['simpleText'],
+            "avatar": item['thumbnail']['thumbnails'][0]['url'],
+            "suscribers": suscribers
+        }
+    return channel
+
+def get_videos_from_search(search):
+    latest = []
+    results = []
+    search = search[1]['response']
+    primaryContents = search['contents']['twoColumnSearchResultsRenderer']['primaryContents']
+    contents = primaryContents['sectionListRenderer']['contents']
+    for content in contents:
+        try:
+            items = content['itemSectionRenderer']['contents']
+        except:
+            continue
+
+    for item in items:
+        try:
+            item['videoRenderer']
+            video = get_video_renderer_item_info(item['videoRenderer'])
+            results.append(video)
+        except KeyError:
+            continue
+
+    # Sometimes Youtube will return an empty query. Try again.        
+    return results
+
+def get_video_renderer_item_info(item):
+    published = ""
+    views = ""
+    isLive = False
+    isUpcoming = False
+
+    thumbnailOverlays = item['thumbnailOverlays']
+    try:
+        if 'UPCOMING' in str(thumbnailOverlays):
+            start_time = item['upcomingEventData']['startTime']
+            isUpcoming = True
+            views = "-"
+            published = "Scheduled"
+    except KeyError:
+        isUpcoming = False
+
+    try:
+        if 'LIVE' in str(thumbnailOverlays):
+            isLive = True
+            try:
+                views = item['viewCountText']['simpleText']
+            except:
+                views = "Live"
+            try:
+                duration = item['lengthText']['simpleText']
+            except:
+                duration = "-"
+            if published != "Scheduled":
+                try:
+                    published = item['publishedTimeText']['simpleText']
+                except KeyError:
+                    published = "None"
+    except:
+        isUpcoming = False
+        isLive = False
+
+    if not isUpcoming and not isLive:
+        views = item['viewCountText']['simpleText']
+        published = item['publishedTimeText']['simpleText']
+        duration = item['lengthText']['simpleText']
+
+    video = {
+        'videoTitle':item['title']['runs'][0]['text'],
+        'description':Markup(str(utils.get_description_snippet_text(item['descriptionSnippet']['runs']))),
+        'views':views,
+        'timeStamp':published,
+        'duration':duration,
+        'channelName':item['ownerText']['runs'][0]['text'],
+        'authorUrl':"/channel/{}".format(item['ownerText']['runs'][0]['navigationEndpoint']['browseEndpoint']['browseId']),
+        'channelId':item['ownerText']['runs'][0]['navigationEndpoint']['browseEndpoint']['browseId'],
+        'id':item['videoId'],
+        'videoUrl':"/watch?v={}".format(item['videoId']),
+        'isLive':isLive,
+        'isUpcoming':isUpcoming,
+        'videoThumb':item['thumbnail']['thumbnails'][0]['url']
+    }
+    return video
 
-    return flask.render_template('search.html',
-        header_playlist_names = local_playlist.get_playlist_names(),
-        query = query,
-        estimated_results = search_info['estimated_results'],
-        estimated_pages = search_info['estimated_pages'],
-        corrections = search_info['corrections'],
-        results = search_info['items'],
-        parameters_dictionary = request.args,
-    )

youtube/util.py

@@ -99,12 +99,7 @@ def decode_content(content, encoding_header):
     return content
 
 
-def bypass_captcha():
+def bypass_captcha(session, response):
-    session = requests.Session()
-    url = "https://youtube.com/watch?v=CvFH_6DNRCY&gl=US&hl=en&has_verified=1&bpctr=9999999999"
-    print("Starting python GET request...")
-    response = session.get(url)
-    print("GET successful!")
     print("vvv COOKIES DICT vvv")
     cookies = session.cookies.get_dict()
     print(cookies)
@@ -195,15 +190,16 @@ def fetch_url_response(url, headers=(), timeout=15, data=None,
         # (in connectionpool.py in urllib3)
         # According to the documentation for urlopen, a redirect counts as a
         # retry. So there are 3 redirects max by default.
-        print("Testing for CAPTCHA python GET request...")
-        r = requests.get(url)
-        print("GET successful!")
 
-        html = BeautifulSoup(str(r.text), "lxml")
+        session = requests.Session()
-        # If there's a captcha and we need to solve it...
+        print("Starting python GET request...")
-        if html.body.find('div', attrs={'class': 'g-recaptcha'}):
+        response = session.get(url)
-            print("ReCaptcha detected! Trying to bypass it.")
+        # Strings that appear when there's a Captcha.
-            bypass_captcha()
+        string_de = "Fülle das folgende Feld aus, um YouTube weiter zu nutzen."
+        string_en = "To continue with your YouTube experience, please fill out the form below."
+        # If there's a captcha, bypass it.
+        if string_de in response.text or string_en in response.text:
+            bypass_captcha(session, response)
 
         if max_redirects:
             retries = urllib3.Retry(3 + max_redirects, redirect=max_redirects)
@@ -464,3 +460,4 @@ def check_gevent_exceptions(*tasks):
     for task in tasks:
         if task.exception:
             raise task.exception
+

youtube/watch.py

@@ -8,11 +8,11 @@ from youtube import util, yt_data_extract
 
 def get_video_sources(info, tor_bypass=False):
     video_sources = []
-    max_resolution = "720"
+    max_resolution = 1080
     for fmt in info['formats']:
         if not all(fmt[attr] for attr in ('quality', 'width', 'ext', 'url')):
             continue
-        if fmt['acodec'] and fmt['vcodec'] and fmt['height'] <= max_resolution:
+        if fmt['acodec'] and fmt['vcodec'] and (fmt['height'] <= max_resolution):
             video_sources.append({
                 'src': fmt['url'],
                 'type': 'video/' + fmt['ext'],
@@ -123,6 +123,24 @@ def get_subtitle_sources(info):
 
     return sources
 
+def decrypt_signatures(info):
+    '''return error string, or False if no errors'''
+    if not yt_data_extract.requires_decryption(info):
+        return False
+    if not info['player_name']:
+        return 'Could not find player name'
+    if not info['base_js']:
+        return 'Failed to find base.js'
+
+    player_name = info['player_name']
+    base_js = util.fetch_url(info['base_js'], debug_name='base.js', report_text='Fetched player ' + player_name)
+    base_js = base_js.decode('utf-8')
+    err = yt_data_extract.extract_decryption_function(info, base_js)
+    if err:
+        return err
+    err = yt_data_extract.decrypt_signatures(info)
+    return err
+
 
 def get_ordered_music_list_attributes(music_list):
     # get the set of attributes which are used by atleast 1 track
@@ -146,9 +164,8 @@ headers = (
     ('X-YouTube-Client-Version', '2.20180830'),
 ) + util.mobile_ua
 def extract_info(video_id, use_invidious, playlist_id=None, index=None):
-    # bpctr=9999999999 will bypass are-you-sure dialogs for controversial
+    # bpctr=9999999999 will bypass are-you-sure dialogs for controversial videos
-    # videos
+    url = 'https://m.youtube.com/watch?v=' + video_id + '&pbj=1&bpctr=9999999999'
-    url = 'https://m.youtube.com/watch?v=' + video_id + '&gl=US&hl=en&has_verified=1&pbj=1&bpctr=9999999999'
     if playlist_id:
         url += '&list=' + playlist_id
     if index:
@@ -173,6 +190,12 @@ def extract_info(video_id, use_invidious, playlist_id=None, index=None):
         url = 'https://www.youtube.com/get_video_info?' + urllib.parse.urlencode(data)
         video_info_page = util.fetch_url(url, debug_name='get_video_info', report_text='Fetched age restriction bypass page').decode('utf-8')
         yt_data_extract.update_with_age_restricted_info(info, video_info_page)
+
+    # signature decryption
+    decryption_error = decrypt_signatures(info)
+    if decryption_error:
+        decryption_error = 'Error decrypting url signatures: ' + decryption_error
+        info['playability_error'] = decryption_error
     # check if urls ready (non-live format) in former livestream
     # urls not ready if all of them have no filesize
     if info['was_live']: