Use new (self implemented) nitter API for Twitter (#140).

app/routes.py

@@ -29,10 +29,14 @@ from youtube_search import YoutubeSearch
 from app import app, db
 from app.forms import LoginForm, RegistrationForm, EmptyForm, SearchForm, ChannelForm
 from app.models import User, twitterPost, ytPost, Post, youtubeFollow, twitterFollow
+
 from youtube import comments, utils, channel as ytch, search as yts
 from youtube import watch as ytwatch
 from youtube import video as ytvid
 
+from nitter import feed as nitterfeed
+from nitter import user as nitteruser
+
 #########################################
 
 #########################################
@@ -81,31 +85,28 @@ def twitter(page=0):
     followCount = len(followingList)
     page = int(page)
     avatarPath = "img/avatars/1.png"
-    posts = []
 
-    nitter_feed_link = config['nitterInstance']
+    followList = []
-    c = len(followingList)
+    for f in followingList:
-    for user in followingList:
+        followList.append(f.username)
-        if c != 0:
+    posts = []
-            nitter_feed_link = nitter_feed_link + "{},".format(user.username)
-            c = c - 1
-        else:
-            nitter_feed_link = nitter_feed_link + user.username
 
     cache_file = glob.glob("app/cache/{}_*".format(current_user.username))
     if (len(cache_file) > 0):
         time_diff = round(time.time() - os.path.getmtime(cache_file[0]))
     else:
         time_diff = 999
+
     # If cache file is more than 1 minute old
     if page == 0 and time_diff > 60:
         if cache_file:
             for f in cache_file:
                 os.remove(f)
-        feed = getFeed(followingList)
+        feed = nitterfeed.get_feed(followList)
         cache_file = "{u}_{d}.json".format(u=current_user.username, d=time.strftime("%Y%m%d-%H%M%S"))
         with open("app/cache/{}".format(cache_file), 'w') as fp:
             json.dump(feed, fp)
+
     # Else, refresh feed
     else:
         try:
@@ -113,14 +114,12 @@ def twitter(page=0):
             with open(cache_file, 'r') as fp:
                 feed = json.load(fp)
         except:
-            feed = getFeed(followingList)
+            feed = nitterfeed.get_feed(followList)
             cache_file = "{u}_{d}.json".format(u=current_user.username, d=time.strftime("%Y%m%d-%H%M%S"))
             with open("app/cache/{}".format(cache_file), 'w') as fp:
                 json.dump(feed, fp)
 
     posts.extend(feed)
-    posts.sort(key=lambda x: datetime.datetime.strptime(x['timeStamp'], '%d/%m/%Y %H:%M:%S'), reverse=True)
-
     # Items range per page
     page_items = page * 16
     offset = page_items + 16
@@ -138,14 +137,8 @@ def twitter(page=0):
         posts = posts[page_items:offset]
     else:
         posts = posts[page_items:]
-
+    return render_template('twitter.html', title='Yotter | Twitter', posts=posts, followedCount=followCount, form=form, config=config,
-    if not posts:
+                           pages=total_pages, init_page=init_page, actual_page=page)
-        profilePic = avatarPath
-    else:
-        profilePic = posts[0]['profilePic']
-    return render_template('twitter.html', title='Yotter | Twitter', posts=posts, avatar=avatarPath,
-                           profilePic=profilePic, followedCount=followCount, form=form, config=config,
-                           pages=total_pages, init_page=init_page, actual_page=page, nitter_link=nitter_feed_link)
 
 
 @app.route('/savePost/<url>', methods=['POST'])
@@ -260,26 +253,39 @@ def search():
     else:
         return render_template('search.html', form=form, config=config)
 
-
 @app.route('/u/<username>')
 @app.route('/<username>')
+@app.route('/<username>/<page>')
 @login_required
-def u(username):
+def u(username, page=1):
+    page=int(page)
     if username == "favicon.ico":
         return redirect(url_for('static', filename='favicons/favicon.ico'))
     form = EmptyForm()
     avatarPath = "img/avatars/{}.png".format(str(random.randint(1, 12)))
-    user = getTwitterUserInfo(username)
+    user = nitteruser.get_user_info(username)
     if not user:
         flash("This user is not on Twitter.")
         return redirect(request.referrer)
 
     posts = []
-    posts.extend(getPosts(username))
+    tweets=nitteruser.get_tweets(username, page)
-    if not posts:
+    if tweets == 'Empty feed':
-        user['profilePic'] = avatarPath
+        posts = False
+    elif tweets == 'Protected feed':
+        posts = 'Protected'
+    else:
+        posts.extend(tweets)
 
-    return render_template('user.html', posts=posts, user=user, form=form, config=config)
+    if page-1 < 0:
+        prev_page = 0
+    else:
+        prev_page = page-1
+    
+    if page > 2:
+        page =2
+
+    return render_template('user.html', posts=posts, user=user, form=form, config=config, page=page, prev_page=prev_page)
 
 
 #########################

app/templates/_twitter_post.html

@@ -19,35 +19,67 @@
                 <span class="category"><i class="retweet icon"></i> {{post.username}}</span>
             {%endif%}
         </div>
-        <div class="description break-word">
+        <div style="margin-bottom: 15px;" class="description break-word">
             <p>{{post.content | safe}}</p>
         </div>
-        <div class="extra content">
+        <div class="content">
-            {% if post.attachedImg %}
+            {% if post.attachedImages %}
-                <a target="_blank" href="{{post.attachedImg}}"><img alt="Image attachment" class="ui centered fluid rounded medium image" src="{{post.attachedImg}}">
+                {%for img in post.attachedImages %}
+                    <a target="_blank" href="{{img}}">
+                        <img alt="Image attachment" class="ui centered fluid rounded medium image" src="{{img}}">
+                    </a>
+                {%endfor%}
             {% endif %}
+            {% if post.attachedVideo %}
+                <div class="ui segment"><p><i class="file video icon"></i> <b>This tweet has an attached video.</b></p></div>
+            {%endif%}
             {% if post.isReply %}
+                {%if post.unavailableReply%}
+                    <div class="ui card">
+                        <div class="content">
+                            <p> This tweet is unavailable. </p>
+                        </div>
+                    </div>
+                {%else%}
                     <div class="ui card">
                         <div class="content">
                             <div class="header"><a href="/{{post.replyingUser}}">{{post.replyingUser}}</a></div>
                             <div class="meta">{{post.replyingUser}}</div>
                             <div class="description break-word">
                                 {{post.replyingTweetContent | safe}}
-                        {% if post.replyAttachedImg %}
-                            <a target="_blank" href="{{post.replyAttachedImg}}"><img alt="Image attachment" class="ui centered fluid rounded medium image" src="{{post.replyAttachedImg}}"></a>
-                        {% endif %}
-                        </div>
-                    </div>
-                </div>
-            {% endif %}
                                 
+                                {% if post.replyAttachedImg %}
+                                    <a target="_blank" href="{{post.replyAttachedImg}}">
+                                        <img alt="Image attachment" class="ui centered fluid rounded medium image" src="{{post.replyAttachedImg}}">
+                                    </a>
+                                {% endif %}
+                            </div>
+                        </div>
+                    </div>
+                {%endif%}
+            {% endif %}
             <p>
                 <form class="ui form" action="{{ url_for('savePost', url=post.url.replace('/', '~')) }}" method="post">
-                    <button type="submit" class="ui icon button">
+                    <button type="submit" class="mini ui icon button">
                         <i class="bookmark outline icon"></i>
                     </button>
                 </form>
             </p>
         </div>
     </div>
+    <div class="extra content">
+        <span class="left floated">
+            <i class="red heart like icon"></i>
+            {{post.likes}}
+            <span> </span>
+            <i class="grey comment icon"></i>
+            {{post.comments}}
+        </span>
+        <span class="right floated">
+            <i class="blue retweet icon"></i>
+                {{post.retweets}}
+            <i class="grey quote left icon"></i>
+            {{post.quotes}}
+        </span>
+    </div>
 </div> <!--End tweet--> 

app/templates/user.html

@@ -1,30 +1,21 @@
 {% extends "base.html" %}
 
 {% block content %}
-    <div class="blue ui centered card">
+    <div class="ui text container center aligned">
-        <div class="content">
+        <div class="ui segments">
-            <div class="center aligned author">
+            <div class="ui centered vertical segment">
-                <img alt="Profile picture" class="ui avatar image" src="{{user.profilePic}}">
+                <h2 class="ui header">
+                <img src="{{user.profilePic}}" class="ui circular image">
+                {{user.profileFullName}} <span style="color:grey;font-size: small;">({{user.profileUsername}})</span>
+                </h2>
             </div>
-            <div class="center aligned header"><a href="https://nitter.net/{{ user.profileUsername.replace('@','') }}">
+            <div class="ui horizontal segments">
-                {%if user.profileFullName%}
+                <div class="ui segment">
-                    {{user.profileFullName}}
+                    <div class="ui centered vertical segment">
-                {%else%}
+                        <p>{{user.profileBio}}</p>
-                    {{user.profileUsername}}
-                {%endif%}
-            </a></div>
-            <div class="center aligned description">
-                    <div class="statistic">
-                        <div class="value">
-                            <i class="users icon"></i>{{user.followers}}
-                        </div>
-                        <div class="label">
-                        Followers
                     </div>
                 </div>
-            </div>
+                <div class="ui segment">
-        </div>
-        <div class="center aligned extra content">
                     {% if not current_user.is_following_tw(user.profileUsername.replace('@','')) %}
                         <p>
                             <form action="{{ url_for('follow', username=user.profileUsername.replace('@','')) }}" method="post">
@@ -42,19 +33,76 @@
                     {% endif %} 
                 </div>
             </div>
+            <div class="ui horizontal segments">
+                <div class="ui segment">
+                    <div class="statistic">
+                        <div class="value">
+                          <b>{{user.followers}}</b>
+                        </div>
+                        <div class="label">
+                            <b>FOLLOWERS</b>
+                        </div>
+                    </div>
+                </div>
+                <div class="ui segment">
+                    <div class="statistic">
+                        <div class="value">
+                          <b>{{user.following}}</b>
+                        </div>
+                        <div class="label">
+                            <b>FOLLOWING</b>
+                        </div>
+                    </div>
+                </div>
+                <div class="ui segment">
+                    <div class="statistic">
+                        <div class="value">
+                          <b>{{user.tweets}}</b>
+                        </div>
+                        <div class="label">
+                            <b>TWEETS</b>
+                        </div>
+                    </div>
+                </div>
+                <div class="ui segment">
+                    <div class="statistic">
+                        <div class="value">
+                          <b>{{user.likes}}</b>
+                        </div>
+                        <div class="label">
+                            <b>LIKES</b>
+                        </div>
+                    </div>
+                </div>
+            </div>
+        </div>
+    </div>
 
-    <div class="text container" id="card-container">
+    <div style="margin-top: 15px;" class="text container" id="card-container">
         {% if not posts %}
-            {% include '_empty_feed.html' %}
+            <div style="margin-top: 20px;" class="ui container center aligned">
+                <h2> <i class="window close outline icon"></i> This feed is empty. </h3>
+            </div>
+        {% elif posts == 'Protected' %}
+            <div style="margin-top: 20px;" class="ui container center aligned">
+                <h2> <i class="lock icon"></i> This account's tweets are protected. </h3>
+            </div>
         {% else %}
             {% for post in posts %}
                     {% include '_twitter_post.html' %}
             {% endfor %}
-        {% endif %}
             <div class="scroller">
                 <a href="#top" class="ui button">
                     <i style="margin: 0;" class="chevron up icon"></i>
                 </a>
             </div>
+            <br>
+            <div class="ui center aligned text container">
+                <a href="/{{user.profileUsername}}/{{prev_page}}"> <button class="ui left attached button"><i class="angle blue left icon"></i></button> </a>
+                <a href="/{{user.profileUsername}}/{{page+1}}"> <button class="right attached ui button"><i class="angle blue right icon"></i></button></a>
             </div>
+            <br>
+        {% endif %}
+    </div>
+
 {% endblock %}

nitter/feed.py

@@ -19,6 +19,7 @@ def get_feed(usernames, daysMaxOld=10, includeRT=True):
     '''
     Returns feed tweets given a set of usernames
     '''
+    print(usernames)
     feedTweets = []
     with FuturesSession() as session:
         futures = [session.get('{instance}{user}'.format(instance=config['nitterInstance'], user=u)) for u in usernames]
@@ -38,5 +39,8 @@ def get_feed(usernames, daysMaxOld=10, includeRT=True):
                     userFeed.append(tweet)
         else:
             userFeed += feed
+    try:
         userFeed.sort(key=lambda x: datetime.datetime.strptime(x['timeStamp'], '%Y-%m-%d %H:%M:%S'), reverse=True)
+    except:
+        return userFeed
     return userFeed

nitter/user.py

@@ -66,8 +66,19 @@ def get_tweets(user, page=1):
         feedPosts = get_feed_tweets(html)
     return feedPosts
 
+def yotterify(text):
+    URLS = ['https://youtube.com']
+    text = str(text)
+    for url in URLS:
+        text.replace(url, "")
+    return text
+
 def get_feed_tweets(html):
     feedPosts = []
+    if 'No items found' in str(html.body):
+        return 'Empty feed'
+    if "This account's tweets are protected." in str(html.body):
+        return 'Protected feed'
     userFeed = html.find_all('div', attrs={'class':'timeline-item'})
     if userFeed != []:
         for post in userFeed[:-1]:
@@ -84,7 +95,7 @@ def get_feed_tweets(html):
             tweet['twitterName'] = post.find('a', attrs={'class':'fullname'}).text
             tweet['timeStamp'] = str(datetime.datetime.strptime(date_time_str, '%d/%m/%Y %H:%M:%S'))
             tweet['date'] = post.find('span', attrs={'class':'tweet-date'}).find('a').text
-            tweet['content'] = Markup(post.find('div',  attrs={'class':'tweet-content'}).decode_contents())
+            tweet['content'] = Markup(yotterify(post.find('div',  attrs={'class':'tweet-content'}).decode_contents().replace("\n", "<br>")))
             
             if post.find('div', attrs={'class':'retweet-header'}):
                 tweet['username'] = post.find('div', attrs={'class':'retweet-header'}).find('div', attrs={'class':'icon-container'}).text
@@ -100,7 +111,17 @@ def get_feed_tweets(html):
             if post.find('div', attrs={'class':'quote'}):
                 tweet['isReply'] = True
                 quote = post.find('div', attrs={'class':'quote'})
+
+                if 'unavailable' in str(quote):
+                    tweet['unavailableReply'] = True
+                else:
+                    tweet['unavailableReply'] = False
+
+                if not tweet['unavailableReply']:
                     if quote.find('div',  attrs={'class':'quote-text'}):
+                        try:
+                            tweet['replyingTweetContent'] = Markup(quote.find('div',  attrs={'class':'quote-text'}).replace("\n", "<br>"))
+                        except:
                             tweet['replyingTweetContent'] = Markup(quote.find('div',  attrs={'class':'quote-text'}))
                         
                     if quote.find('a', attrs={'class':'still-image'}):
@@ -110,7 +131,6 @@ def get_feed_tweets(html):
                             img = BeautifulSoup(str(img), "lxml")
                             url = config['nitterInstance'] + img.find('a')['href'][1:]
                             tweet['replyAttachedImages'].append(url)
-
                     tweet['replyingUser']=quote.find('a',  attrs={'class':'username'}).text
                     post.find('div', attrs={'class':'quote'}).decompose()
             else:
@@ -129,13 +149,25 @@ def get_feed_tweets(html):
                 else:
                     tweet['attachedImages'] = False
                 # Videos    
-                if post.find('div', attrs={'gallery-video'}):
+                if post.find('div', attrs={'attachments'}).find('div', attrs={'gallery-video'}):
                     tweet['attachedVideo'] = True
                 else:
                     tweet['attachedVideo'] = False
             else:
                 tweet['attachedVideo'] = False
                 tweet['attachedImages'] = False
+
+            if post.find('div', attrs={'class':'tweet-stats'}):
+                stats = post.find('div', attrs={'class':'tweet-stats'}).find_all('span', attrs={'class':'tweet-stat'})
+                for stat in stats:
+                    if 'comment' in str(stat):
+                        tweet['comments'] = stat.find('div',attrs={'class':'icon-container'}).text
+                    elif 'retweet' in str(stat):
+                        tweet['retweets'] = stat.find('div',attrs={'class':'icon-container'}).text
+                    elif 'heart' in str(stat):
+                        tweet['likes'] = stat.find('div',attrs={'class':'icon-container'}).text
+                    else:
+                        tweet['quotes'] =  stat.find('div',attrs={'class':'icon-container'}).text              
             feedPosts.append(tweet)
     else:
         return {"emptyFeed": True}