From fac46ee8531a2899d5e7de169ba117b8107830d1 Mon Sep 17 00:00:00 2001 From: pluja Date: Mon, 9 Nov 2020 15:39:30 +0100 Subject: [PATCH] Use new (self implemented) nitter API for Twitter (#140). --- app/routes.py | 62 ++++++++------- app/templates/_twitter_post.html | 66 +++++++++++----- app/templates/user.html | 126 +++++++++++++++++++++---------- nitter/feed.py | 6 +- nitter/user.py | 60 +++++++++++---- 5 files changed, 221 insertions(+), 99 deletions(-) diff --git a/app/routes.py b/app/routes.py index 16440a2..a54b9c6 100644 --- a/app/routes.py +++ b/app/routes.py @@ -29,10 +29,14 @@ from youtube_search import YoutubeSearch from app import app, db from app.forms import LoginForm, RegistrationForm, EmptyForm, SearchForm, ChannelForm from app.models import User, twitterPost, ytPost, Post, youtubeFollow, twitterFollow + from youtube import comments, utils, channel as ytch, search as yts from youtube import watch as ytwatch from youtube import video as ytvid +from nitter import feed as nitterfeed +from nitter import user as nitteruser + ######################################### ######################################### @@ -81,31 +85,28 @@ def twitter(page=0): followCount = len(followingList) page = int(page) avatarPath = "img/avatars/1.png" - posts = [] - nitter_feed_link = config['nitterInstance'] - c = len(followingList) - for user in followingList: - if c != 0: - nitter_feed_link = nitter_feed_link + "{},".format(user.username) - c = c - 1 - else: - nitter_feed_link = nitter_feed_link + user.username + followList = [] + for f in followingList: + followList.append(f.username) + posts = [] cache_file = glob.glob("app/cache/{}_*".format(current_user.username)) if (len(cache_file) > 0): time_diff = round(time.time() - os.path.getmtime(cache_file[0])) else: time_diff = 999 + # If cache file is more than 1 minute old if page == 0 and time_diff > 60: if cache_file: for f in cache_file: os.remove(f) - feed = getFeed(followingList) + feed = nitterfeed.get_feed(followList) cache_file = "{u}_{d}.json".format(u=current_user.username, d=time.strftime("%Y%m%d-%H%M%S")) with open("app/cache/{}".format(cache_file), 'w') as fp: json.dump(feed, fp) + # Else, refresh feed else: try: @@ -113,14 +114,12 @@ def twitter(page=0): with open(cache_file, 'r') as fp: feed = json.load(fp) except: - feed = getFeed(followingList) + feed = nitterfeed.get_feed(followList) cache_file = "{u}_{d}.json".format(u=current_user.username, d=time.strftime("%Y%m%d-%H%M%S")) with open("app/cache/{}".format(cache_file), 'w') as fp: json.dump(feed, fp) posts.extend(feed) - posts.sort(key=lambda x: datetime.datetime.strptime(x['timeStamp'], '%d/%m/%Y %H:%M:%S'), reverse=True) - # Items range per page page_items = page * 16 offset = page_items + 16 @@ -138,14 +137,8 @@ def twitter(page=0): posts = posts[page_items:offset] else: posts = posts[page_items:] - - if not posts: - profilePic = avatarPath - else: - profilePic = posts[0]['profilePic'] - return render_template('twitter.html', title='Yotter | Twitter', posts=posts, avatar=avatarPath, - profilePic=profilePic, followedCount=followCount, form=form, config=config, - pages=total_pages, init_page=init_page, actual_page=page, nitter_link=nitter_feed_link) + return render_template('twitter.html', title='Yotter | Twitter', posts=posts, followedCount=followCount, form=form, config=config, + pages=total_pages, init_page=init_page, actual_page=page) @app.route('/savePost/', methods=['POST']) @@ -260,26 +253,39 @@ def search(): else: return render_template('search.html', form=form, config=config) - @app.route('/u/') @app.route('/') +@app.route('//') @login_required -def u(username): +def u(username, page=1): + page=int(page) if username == "favicon.ico": return redirect(url_for('static', filename='favicons/favicon.ico')) form = EmptyForm() avatarPath = "img/avatars/{}.png".format(str(random.randint(1, 12))) - user = getTwitterUserInfo(username) + user = nitteruser.get_user_info(username) if not user: flash("This user is not on Twitter.") return redirect(request.referrer) posts = [] - posts.extend(getPosts(username)) - if not posts: - user['profilePic'] = avatarPath + tweets=nitteruser.get_tweets(username, page) + if tweets == 'Empty feed': + posts = False + elif tweets == 'Protected feed': + posts = 'Protected' + else: + posts.extend(tweets) - return render_template('user.html', posts=posts, user=user, form=form, config=config) + if page-1 < 0: + prev_page = 0 + else: + prev_page = page-1 + + if page > 2: + page =2 + + return render_template('user.html', posts=posts, user=user, form=form, config=config, page=page, prev_page=prev_page) ######################### diff --git a/app/templates/_twitter_post.html b/app/templates/_twitter_post.html index cba6368..6826f3f 100644 --- a/app/templates/_twitter_post.html +++ b/app/templates/_twitter_post.html @@ -19,35 +19,67 @@ {{post.username}} {%endif%} -
+

{{post.content | safe}}

-
- {% if post.attachedImg %} - Image attachment +
+ {% if post.attachedImages %} + {%for img in post.attachedImages %} + + Image attachment + + {%endfor%} {% endif %} + {% if post.attachedVideo %} +

This tweet has an attached video.

+ {%endif%} {% if post.isReply %} -
-
-
{{post.replyingUser}}
-
{{post.replyingUser}}
-
- {{post.replyingTweetContent | safe}} - {% if post.replyAttachedImg %} - Image attachment - {% endif %} + {%if post.unavailableReply%} +
+
+

This tweet is unavailable.

-
+ {%else%} +
+
+
{{post.replyingUser}}
+
{{post.replyingUser}}
+
+ {{post.replyingTweetContent | safe}} + + {% if post.replyAttachedImg %} + + Image attachment + + {% endif %} +
+
+
+ {%endif%} {% endif %} -

- +

+
+ + + {{post.likes}} + + + {{post.comments}} + + + + {{post.retweets}} + + {{post.quotes}} + +
\ No newline at end of file diff --git a/app/templates/user.html b/app/templates/user.html index cfeee65..a425ac5 100644 --- a/app/templates/user.html +++ b/app/templates/user.html @@ -1,60 +1,108 @@ {% extends "base.html" %} {% block content %} -
-
-
- Profile picture +
+
+
+

+ + {{user.profileFullName}} ({{user.profileUsername}}) +

-
- {%if user.profileFullName%} - {{user.profileFullName}} - {%else%} - {{user.profileUsername}} - {%endif%} -
-
+
+
+
+

{{user.profileBio}}

+
+
+
+ {% if not current_user.is_following_tw(user.profileUsername.replace('@','')) %} +

+

+ {{ form.hidden_tag() }} + {{ form.submit(value='Follow') }} +
+

+ {% else %} +

+

+ {{ form.hidden_tag() }} + {{ form.submit(value='Unfollow') }} +
+

+ {% endif %} +
+
+
+
- {{user.followers}} + {{user.followers}}
- Followers + FOLLOWERS
-
+
+
+
+
+
+ {{user.following}} +
+
+ FOLLOWING +
+
+
+
+
+
+ {{user.tweets}} +
+
+ TWEETS +
+
+
+
+
+
+ {{user.likes}} +
+
+ LIKES +
+
+
-
- {% if not current_user.is_following_tw(user.profileUsername.replace('@','')) %} -

-

- {{ form.hidden_tag() }} - {{ form.submit(value='Follow') }} -
-

- {% else %} -

-

- {{ form.hidden_tag() }} - {{ form.submit(value='Unfollow') }} -
-

- {% endif %} -
-
+
{% if not posts %} - {% include '_empty_feed.html' %} +
+

This feed is empty.

+
+ {% elif posts == 'Protected' %} +
+

This account's tweets are protected.

+
{% else %} {% for post in posts %} {% include '_twitter_post.html' %} {% endfor %} +
+ + + +
+
+
+ + +
+
{% endif %} -
- - - -
+ {% endblock %} \ No newline at end of file diff --git a/nitter/feed.py b/nitter/feed.py index e4dabf6..d6c5e62 100644 --- a/nitter/feed.py +++ b/nitter/feed.py @@ -19,6 +19,7 @@ def get_feed(usernames, daysMaxOld=10, includeRT=True): ''' Returns feed tweets given a set of usernames ''' + print(usernames) feedTweets = [] with FuturesSession() as session: futures = [session.get('{instance}{user}'.format(instance=config['nitterInstance'], user=u)) for u in usernames] @@ -38,5 +39,8 @@ def get_feed(usernames, daysMaxOld=10, includeRT=True): userFeed.append(tweet) else: userFeed += feed - userFeed.sort(key=lambda x: datetime.datetime.strptime(x['timeStamp'], '%Y-%m-%d %H:%M:%S'), reverse=True) + try: + userFeed.sort(key=lambda x: datetime.datetime.strptime(x['timeStamp'], '%Y-%m-%d %H:%M:%S'), reverse=True) + except: + return userFeed return userFeed \ No newline at end of file diff --git a/nitter/user.py b/nitter/user.py index 51e913e..542625e 100644 --- a/nitter/user.py +++ b/nitter/user.py @@ -66,8 +66,19 @@ def get_tweets(user, page=1): feedPosts = get_feed_tweets(html) return feedPosts +def yotterify(text): + URLS = ['https://youtube.com'] + text = str(text) + for url in URLS: + text.replace(url, "") + return text + def get_feed_tweets(html): feedPosts = [] + if 'No items found' in str(html.body): + return 'Empty feed' + if "This account's tweets are protected." in str(html.body): + return 'Protected feed' userFeed = html.find_all('div', attrs={'class':'timeline-item'}) if userFeed != []: for post in userFeed[:-1]: @@ -84,7 +95,7 @@ def get_feed_tweets(html): tweet['twitterName'] = post.find('a', attrs={'class':'fullname'}).text tweet['timeStamp'] = str(datetime.datetime.strptime(date_time_str, '%d/%m/%Y %H:%M:%S')) tweet['date'] = post.find('span', attrs={'class':'tweet-date'}).find('a').text - tweet['content'] = Markup(post.find('div', attrs={'class':'tweet-content'}).decode_contents()) + tweet['content'] = Markup(yotterify(post.find('div', attrs={'class':'tweet-content'}).decode_contents().replace("\n", "
"))) if post.find('div', attrs={'class':'retweet-header'}): tweet['username'] = post.find('div', attrs={'class':'retweet-header'}).find('div', attrs={'class':'icon-container'}).text @@ -100,19 +111,28 @@ def get_feed_tweets(html): if post.find('div', attrs={'class':'quote'}): tweet['isReply'] = True quote = post.find('div', attrs={'class':'quote'}) - if quote.find('div', attrs={'class':'quote-text'}): - tweet['replyingTweetContent'] = Markup(quote.find('div', attrs={'class':'quote-text'})) - - if quote.find('a', attrs={'class':'still-image'}): - tweet['replyAttachedImages'] = [] - images = quote.find_all('a', attrs={'class':'still-image'}) - for img in images: - img = BeautifulSoup(str(img), "lxml") - url = config['nitterInstance'] + img.find('a')['href'][1:] - tweet['replyAttachedImages'].append(url) - tweet['replyingUser']=quote.find('a', attrs={'class':'username'}).text - post.find('div', attrs={'class':'quote'}).decompose() + if 'unavailable' in str(quote): + tweet['unavailableReply'] = True + else: + tweet['unavailableReply'] = False + + if not tweet['unavailableReply']: + if quote.find('div', attrs={'class':'quote-text'}): + try: + tweet['replyingTweetContent'] = Markup(quote.find('div', attrs={'class':'quote-text'}).replace("\n", "
")) + except: + tweet['replyingTweetContent'] = Markup(quote.find('div', attrs={'class':'quote-text'})) + + if quote.find('a', attrs={'class':'still-image'}): + tweet['replyAttachedImages'] = [] + images = quote.find_all('a', attrs={'class':'still-image'}) + for img in images: + img = BeautifulSoup(str(img), "lxml") + url = config['nitterInstance'] + img.find('a')['href'][1:] + tweet['replyAttachedImages'].append(url) + tweet['replyingUser']=quote.find('a', attrs={'class':'username'}).text + post.find('div', attrs={'class':'quote'}).decompose() else: tweet['isReply'] = False @@ -129,13 +149,25 @@ def get_feed_tweets(html): else: tweet['attachedImages'] = False # Videos - if post.find('div', attrs={'gallery-video'}): + if post.find('div', attrs={'attachments'}).find('div', attrs={'gallery-video'}): tweet['attachedVideo'] = True else: tweet['attachedVideo'] = False else: tweet['attachedVideo'] = False tweet['attachedImages'] = False + + if post.find('div', attrs={'class':'tweet-stats'}): + stats = post.find('div', attrs={'class':'tweet-stats'}).find_all('span', attrs={'class':'tweet-stat'}) + for stat in stats: + if 'comment' in str(stat): + tweet['comments'] = stat.find('div',attrs={'class':'icon-container'}).text + elif 'retweet' in str(stat): + tweet['retweets'] = stat.find('div',attrs={'class':'icon-container'}).text + elif 'heart' in str(stat): + tweet['likes'] = stat.find('div',attrs={'class':'icon-container'}).text + else: + tweet['quotes'] = stat.find('div',attrs={'class':'icon-container'}).text feedPosts.append(tweet) else: return {"emptyFeed": True}