Use new (self implemented) nitter API for Twitter (#140).
This commit is contained in:
pluja
parent
92689b954c
commit
fac46ee853
@ -29,10 +29,14 @@ from youtube_search import YoutubeSearch
|
||||
from app import app, db
|
||||
from app.forms import LoginForm, RegistrationForm, EmptyForm, SearchForm, ChannelForm
|
||||
from app.models import User, twitterPost, ytPost, Post, youtubeFollow, twitterFollow
|
||||
|
||||
from youtube import comments, utils, channel as ytch, search as yts
|
||||
from youtube import watch as ytwatch
|
||||
from youtube import video as ytvid
|
||||
|
||||
from nitter import feed as nitterfeed
|
||||
from nitter import user as nitteruser
|
||||
|
||||
#########################################
|
||||
|
||||
#########################################
|
||||
@ -81,31 +85,28 @@ def twitter(page=0):
|
||||
followCount = len(followingList)
|
||||
page = int(page)
|
||||
avatarPath = "img/avatars/1.png"
|
||||
posts = []
|
||||
|
||||
nitter_feed_link = config['nitterInstance']
|
||||
c = len(followingList)
|
||||
for user in followingList:
|
||||
if c != 0:
|
||||
nitter_feed_link = nitter_feed_link + "{},".format(user.username)
|
||||
c = c - 1
|
||||
else:
|
||||
nitter_feed_link = nitter_feed_link + user.username
|
||||
followList = []
|
||||
for f in followingList:
|
||||
followList.append(f.username)
|
||||
posts = []
|
||||
|
||||
cache_file = glob.glob("app/cache/{}_*".format(current_user.username))
|
||||
if (len(cache_file) > 0):
|
||||
time_diff = round(time.time() - os.path.getmtime(cache_file[0]))
|
||||
else:
|
||||
time_diff = 999
|
||||
|
||||
# If cache file is more than 1 minute old
|
||||
if page == 0 and time_diff > 60:
|
||||
if cache_file:
|
||||
for f in cache_file:
|
||||
os.remove(f)
|
||||
feed = getFeed(followingList)
|
||||
feed = nitterfeed.get_feed(followList)
|
||||
cache_file = "{u}_{d}.json".format(u=current_user.username, d=time.strftime("%Y%m%d-%H%M%S"))
|
||||
with open("app/cache/{}".format(cache_file), 'w') as fp:
|
||||
json.dump(feed, fp)
|
||||
|
||||
# Else, refresh feed
|
||||
else:
|
||||
try:
|
||||
@ -113,14 +114,12 @@ def twitter(page=0):
|
||||
with open(cache_file, 'r') as fp:
|
||||
feed = json.load(fp)
|
||||
except:
|
||||
feed = getFeed(followingList)
|
||||
feed = nitterfeed.get_feed(followList)
|
||||
cache_file = "{u}_{d}.json".format(u=current_user.username, d=time.strftime("%Y%m%d-%H%M%S"))
|
||||
with open("app/cache/{}".format(cache_file), 'w') as fp:
|
||||
json.dump(feed, fp)
|
||||
|
||||
posts.extend(feed)
|
||||
posts.sort(key=lambda x: datetime.datetime.strptime(x['timeStamp'], '%d/%m/%Y %H:%M:%S'), reverse=True)
|
||||
|
||||
# Items range per page
|
||||
page_items = page * 16
|
||||
offset = page_items + 16
|
||||
@ -138,14 +137,8 @@ def twitter(page=0):
|
||||
posts = posts[page_items:offset]
|
||||
else:
|
||||
posts = posts[page_items:]
|
||||
|
||||
if not posts:
|
||||
profilePic = avatarPath
|
||||
else:
|
||||
profilePic = posts[0]['profilePic']
|
||||
return render_template('twitter.html', title='Yotter | Twitter', posts=posts, avatar=avatarPath,
|
||||
profilePic=profilePic, followedCount=followCount, form=form, config=config,
|
||||
pages=total_pages, init_page=init_page, actual_page=page, nitter_link=nitter_feed_link)
|
||||
return render_template('twitter.html', title='Yotter | Twitter', posts=posts, followedCount=followCount, form=form, config=config,
|
||||
pages=total_pages, init_page=init_page, actual_page=page)
|
||||
|
||||
|
||||
@app.route('/savePost/<url>', methods=['POST'])
|
||||
@ -260,26 +253,39 @@ def search():
|
||||
else:
|
||||
return render_template('search.html', form=form, config=config)
|
||||
|
||||
|
||||
@app.route('/u/<username>')
|
||||
@app.route('/<username>')
|
||||
@app.route('/<username>/<page>')
|
||||
@login_required
|
||||
def u(username):
|
||||
def u(username, page=1):
|
||||
page=int(page)
|
||||
if username == "favicon.ico":
|
||||
return redirect(url_for('static', filename='favicons/favicon.ico'))
|
||||
form = EmptyForm()
|
||||
avatarPath = "img/avatars/{}.png".format(str(random.randint(1, 12)))
|
||||
user = getTwitterUserInfo(username)
|
||||
user = nitteruser.get_user_info(username)
|
||||
if not user:
|
||||
flash("This user is not on Twitter.")
|
||||
return redirect(request.referrer)
|
||||
|
||||
posts = []
|
||||
posts.extend(getPosts(username))
|
||||
if not posts:
|
||||
user['profilePic'] = avatarPath
|
||||
tweets=nitteruser.get_tweets(username, page)
|
||||
if tweets == 'Empty feed':
|
||||
posts = False
|
||||
elif tweets == 'Protected feed':
|
||||
posts = 'Protected'
|
||||
else:
|
||||
posts.extend(tweets)
|
||||
|
||||
return render_template('user.html', posts=posts, user=user, form=form, config=config)
|
||||
if page-1 < 0:
|
||||
prev_page = 0
|
||||
else:
|
||||
prev_page = page-1
|
||||
|
||||
if page > 2:
|
||||
page =2
|
||||
|
||||
return render_template('user.html', posts=posts, user=user, form=form, config=config, page=page, prev_page=prev_page)
|
||||
|
||||
|
||||
#########################
|
||||
|
||||
@ -19,35 +19,67 @@
|
||||
<span class="category"><i class="retweet icon"></i> {{post.username}}</span>
|
||||
{%endif%}
|
||||
</div>
|
||||
<div class="description break-word">
|
||||
<div style="margin-bottom: 15px;" class="description break-word">
|
||||
<p>{{post.content | safe}}</p>
|
||||
</div>
|
||||
<div class="extra content">
|
||||
{% if post.attachedImg %}
|
||||
<a target="_blank" href="{{post.attachedImg}}"><img alt="Image attachment" class="ui centered fluid rounded medium image" src="{{post.attachedImg}}">
|
||||
<div class="content">
|
||||
{% if post.attachedImages %}
|
||||
{%for img in post.attachedImages %}
|
||||
<a target="_blank" href="{{img}}">
|
||||
<img alt="Image attachment" class="ui centered fluid rounded medium image" src="{{img}}">
|
||||
</a>
|
||||
{%endfor%}
|
||||
{% endif %}
|
||||
{% if post.attachedVideo %}
|
||||
<div class="ui segment"><p><i class="file video icon"></i> <b>This tweet has an attached video.</b></p></div>
|
||||
{%endif%}
|
||||
{% if post.isReply %}
|
||||
<div class="ui card">
|
||||
<div class="content">
|
||||
<div class="header"><a href="/{{post.replyingUser}}">{{post.replyingUser}}</a></div>
|
||||
<div class="meta">{{post.replyingUser}}</div>
|
||||
<div class="description break-word">
|
||||
{{post.replyingTweetContent | safe}}
|
||||
{% if post.replyAttachedImg %}
|
||||
<a target="_blank" href="{{post.replyAttachedImg}}"><img alt="Image attachment" class="ui centered fluid rounded medium image" src="{{post.replyAttachedImg}}"></a>
|
||||
{% endif %}
|
||||
{%if post.unavailableReply%}
|
||||
<div class="ui card">
|
||||
<div class="content">
|
||||
<p> This tweet is unavailable. </p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
{%else%}
|
||||
<div class="ui card">
|
||||
<div class="content">
|
||||
<div class="header"><a href="/{{post.replyingUser}}">{{post.replyingUser}}</a></div>
|
||||
<div class="meta">{{post.replyingUser}}</div>
|
||||
<div class="description break-word">
|
||||
{{post.replyingTweetContent | safe}}
|
||||
|
||||
{% if post.replyAttachedImg %}
|
||||
<a target="_blank" href="{{post.replyAttachedImg}}">
|
||||
<img alt="Image attachment" class="ui centered fluid rounded medium image" src="{{post.replyAttachedImg}}">
|
||||
</a>
|
||||
{% endif %}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
{%endif%}
|
||||
{% endif %}
|
||||
|
||||
<p>
|
||||
<form class="ui form" action="{{ url_for('savePost', url=post.url.replace('/', '~')) }}" method="post">
|
||||
<button type="submit" class="ui icon button">
|
||||
<button type="submit" class="mini ui icon button">
|
||||
<i class="bookmark outline icon"></i>
|
||||
</button>
|
||||
</button>
|
||||
</form>
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
<div class="extra content">
|
||||
<span class="left floated">
|
||||
<i class="red heart like icon"></i>
|
||||
{{post.likes}}
|
||||
<span> </span>
|
||||
<i class="grey comment icon"></i>
|
||||
{{post.comments}}
|
||||
</span>
|
||||
<span class="right floated">
|
||||
<i class="blue retweet icon"></i>
|
||||
{{post.retweets}}
|
||||
<i class="grey quote left icon"></i>
|
||||
{{post.quotes}}
|
||||
</span>
|
||||
</div>
|
||||
</div> <!--End tweet-->
|
||||
@ -1,60 +1,108 @@
|
||||
{% extends "base.html" %}
|
||||
|
||||
{% block content %}
|
||||
<div class="blue ui centered card">
|
||||
<div class="content">
|
||||
<div class="center aligned author">
|
||||
<img alt="Profile picture" class="ui avatar image" src="{{user.profilePic}}">
|
||||
<div class="ui text container center aligned">
|
||||
<div class="ui segments">
|
||||
<div class="ui centered vertical segment">
|
||||
<h2 class="ui header">
|
||||
<img src="{{user.profilePic}}" class="ui circular image">
|
||||
{{user.profileFullName}} <span style="color:grey;font-size: small;">({{user.profileUsername}})</span>
|
||||
</h2>
|
||||
</div>
|
||||
<div class="center aligned header"><a href="https://nitter.net/{{ user.profileUsername.replace('@','') }}">
|
||||
{%if user.profileFullName%}
|
||||
{{user.profileFullName}}
|
||||
{%else%}
|
||||
{{user.profileUsername}}
|
||||
{%endif%}
|
||||
</a></div>
|
||||
<div class="center aligned description">
|
||||
<div class="ui horizontal segments">
|
||||
<div class="ui segment">
|
||||
<div class="ui centered vertical segment">
|
||||
<p>{{user.profileBio}}</p>
|
||||
</div>
|
||||
</div>
|
||||
<div class="ui segment">
|
||||
{% if not current_user.is_following_tw(user.profileUsername.replace('@','')) %}
|
||||
<p>
|
||||
<form action="{{ url_for('follow', username=user.profileUsername.replace('@','')) }}" method="post">
|
||||
{{ form.hidden_tag() }}
|
||||
{{ form.submit(value='Follow') }}
|
||||
</form>
|
||||
</p>
|
||||
{% else %}
|
||||
<p>
|
||||
<form action="{{ url_for('unfollow', username=user.profileUsername.replace('@','')) }}" method="post">
|
||||
{{ form.hidden_tag() }}
|
||||
{{ form.submit(value='Unfollow') }}
|
||||
</form>
|
||||
</p>
|
||||
{% endif %}
|
||||
</div>
|
||||
</div>
|
||||
<div class="ui horizontal segments">
|
||||
<div class="ui segment">
|
||||
<div class="statistic">
|
||||
<div class="value">
|
||||
<i class="users icon"></i>{{user.followers}}
|
||||
<b>{{user.followers}}</b>
|
||||
</div>
|
||||
<div class="label">
|
||||
Followers
|
||||
<b>FOLLOWERS</b>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="ui segment">
|
||||
<div class="statistic">
|
||||
<div class="value">
|
||||
<b>{{user.following}}</b>
|
||||
</div>
|
||||
<div class="label">
|
||||
<b>FOLLOWING</b>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="ui segment">
|
||||
<div class="statistic">
|
||||
<div class="value">
|
||||
<b>{{user.tweets}}</b>
|
||||
</div>
|
||||
<div class="label">
|
||||
<b>TWEETS</b>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="ui segment">
|
||||
<div class="statistic">
|
||||
<div class="value">
|
||||
<b>{{user.likes}}</b>
|
||||
</div>
|
||||
<div class="label">
|
||||
<b>LIKES</b>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="center aligned extra content">
|
||||
{% if not current_user.is_following_tw(user.profileUsername.replace('@','')) %}
|
||||
<p>
|
||||
<form action="{{ url_for('follow', username=user.profileUsername.replace('@','')) }}" method="post">
|
||||
{{ form.hidden_tag() }}
|
||||
{{ form.submit(value='Follow') }}
|
||||
</form>
|
||||
</p>
|
||||
{% else %}
|
||||
<p>
|
||||
<form action="{{ url_for('unfollow', username=user.profileUsername.replace('@','')) }}" method="post">
|
||||
{{ form.hidden_tag() }}
|
||||
{{ form.submit(value='Unfollow') }}
|
||||
</form>
|
||||
</p>
|
||||
{% endif %}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="text container" id="card-container">
|
||||
<div style="margin-top: 15px;" class="text container" id="card-container">
|
||||
{% if not posts %}
|
||||
{% include '_empty_feed.html' %}
|
||||
<div style="margin-top: 20px;" class="ui container center aligned">
|
||||
<h2> <i class="window close outline icon"></i> This feed is empty. </h3>
|
||||
</div>
|
||||
{% elif posts == 'Protected' %}
|
||||
<div style="margin-top: 20px;" class="ui container center aligned">
|
||||
<h2> <i class="lock icon"></i> This account's tweets are protected. </h3>
|
||||
</div>
|
||||
{% else %}
|
||||
{% for post in posts %}
|
||||
{% include '_twitter_post.html' %}
|
||||
{% endfor %}
|
||||
<div class="scroller">
|
||||
<a href="#top" class="ui button">
|
||||
<i style="margin: 0;" class="chevron up icon"></i>
|
||||
</a>
|
||||
</div>
|
||||
<br>
|
||||
<div class="ui center aligned text container">
|
||||
<a href="/{{user.profileUsername}}/{{prev_page}}"> <button class="ui left attached button"><i class="angle blue left icon"></i></button> </a>
|
||||
<a href="/{{user.profileUsername}}/{{page+1}}"> <button class="right attached ui button"><i class="angle blue right icon"></i></button></a>
|
||||
</div>
|
||||
<br>
|
||||
{% endif %}
|
||||
<div class="scroller">
|
||||
<a href="#top" class="ui button">
|
||||
<i style="margin: 0;" class="chevron up icon"></i>
|
||||
</a>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{% endblock %}
|
||||
@ -19,6 +19,7 @@ def get_feed(usernames, daysMaxOld=10, includeRT=True):
|
||||
'''
|
||||
Returns feed tweets given a set of usernames
|
||||
'''
|
||||
print(usernames)
|
||||
feedTweets = []
|
||||
with FuturesSession() as session:
|
||||
futures = [session.get('{instance}{user}'.format(instance=config['nitterInstance'], user=u)) for u in usernames]
|
||||
@ -38,5 +39,8 @@ def get_feed(usernames, daysMaxOld=10, includeRT=True):
|
||||
userFeed.append(tweet)
|
||||
else:
|
||||
userFeed += feed
|
||||
userFeed.sort(key=lambda x: datetime.datetime.strptime(x['timeStamp'], '%Y-%m-%d %H:%M:%S'), reverse=True)
|
||||
try:
|
||||
userFeed.sort(key=lambda x: datetime.datetime.strptime(x['timeStamp'], '%Y-%m-%d %H:%M:%S'), reverse=True)
|
||||
except:
|
||||
return userFeed
|
||||
return userFeed
|
||||
@ -66,8 +66,19 @@ def get_tweets(user, page=1):
|
||||
feedPosts = get_feed_tweets(html)
|
||||
return feedPosts
|
||||
|
||||
def yotterify(text):
|
||||
URLS = ['https://youtube.com']
|
||||
text = str(text)
|
||||
for url in URLS:
|
||||
text.replace(url, "")
|
||||
return text
|
||||
|
||||
def get_feed_tweets(html):
|
||||
feedPosts = []
|
||||
if 'No items found' in str(html.body):
|
||||
return 'Empty feed'
|
||||
if "This account's tweets are protected." in str(html.body):
|
||||
return 'Protected feed'
|
||||
userFeed = html.find_all('div', attrs={'class':'timeline-item'})
|
||||
if userFeed != []:
|
||||
for post in userFeed[:-1]:
|
||||
@ -84,7 +95,7 @@ def get_feed_tweets(html):
|
||||
tweet['twitterName'] = post.find('a', attrs={'class':'fullname'}).text
|
||||
tweet['timeStamp'] = str(datetime.datetime.strptime(date_time_str, '%d/%m/%Y %H:%M:%S'))
|
||||
tweet['date'] = post.find('span', attrs={'class':'tweet-date'}).find('a').text
|
||||
tweet['content'] = Markup(post.find('div', attrs={'class':'tweet-content'}).decode_contents())
|
||||
tweet['content'] = Markup(yotterify(post.find('div', attrs={'class':'tweet-content'}).decode_contents().replace("\n", "<br>")))
|
||||
|
||||
if post.find('div', attrs={'class':'retweet-header'}):
|
||||
tweet['username'] = post.find('div', attrs={'class':'retweet-header'}).find('div', attrs={'class':'icon-container'}).text
|
||||
@ -100,19 +111,28 @@ def get_feed_tweets(html):
|
||||
if post.find('div', attrs={'class':'quote'}):
|
||||
tweet['isReply'] = True
|
||||
quote = post.find('div', attrs={'class':'quote'})
|
||||
if quote.find('div', attrs={'class':'quote-text'}):
|
||||
tweet['replyingTweetContent'] = Markup(quote.find('div', attrs={'class':'quote-text'}))
|
||||
|
||||
if quote.find('a', attrs={'class':'still-image'}):
|
||||
tweet['replyAttachedImages'] = []
|
||||
images = quote.find_all('a', attrs={'class':'still-image'})
|
||||
for img in images:
|
||||
img = BeautifulSoup(str(img), "lxml")
|
||||
url = config['nitterInstance'] + img.find('a')['href'][1:]
|
||||
tweet['replyAttachedImages'].append(url)
|
||||
|
||||
tweet['replyingUser']=quote.find('a', attrs={'class':'username'}).text
|
||||
post.find('div', attrs={'class':'quote'}).decompose()
|
||||
if 'unavailable' in str(quote):
|
||||
tweet['unavailableReply'] = True
|
||||
else:
|
||||
tweet['unavailableReply'] = False
|
||||
|
||||
if not tweet['unavailableReply']:
|
||||
if quote.find('div', attrs={'class':'quote-text'}):
|
||||
try:
|
||||
tweet['replyingTweetContent'] = Markup(quote.find('div', attrs={'class':'quote-text'}).replace("\n", "<br>"))
|
||||
except:
|
||||
tweet['replyingTweetContent'] = Markup(quote.find('div', attrs={'class':'quote-text'}))
|
||||
|
||||
if quote.find('a', attrs={'class':'still-image'}):
|
||||
tweet['replyAttachedImages'] = []
|
||||
images = quote.find_all('a', attrs={'class':'still-image'})
|
||||
for img in images:
|
||||
img = BeautifulSoup(str(img), "lxml")
|
||||
url = config['nitterInstance'] + img.find('a')['href'][1:]
|
||||
tweet['replyAttachedImages'].append(url)
|
||||
tweet['replyingUser']=quote.find('a', attrs={'class':'username'}).text
|
||||
post.find('div', attrs={'class':'quote'}).decompose()
|
||||
else:
|
||||
tweet['isReply'] = False
|
||||
|
||||
@ -129,13 +149,25 @@ def get_feed_tweets(html):
|
||||
else:
|
||||
tweet['attachedImages'] = False
|
||||
# Videos
|
||||
if post.find('div', attrs={'gallery-video'}):
|
||||
if post.find('div', attrs={'attachments'}).find('div', attrs={'gallery-video'}):
|
||||
tweet['attachedVideo'] = True
|
||||
else:
|
||||
tweet['attachedVideo'] = False
|
||||
else:
|
||||
tweet['attachedVideo'] = False
|
||||
tweet['attachedImages'] = False
|
||||
|
||||
if post.find('div', attrs={'class':'tweet-stats'}):
|
||||
stats = post.find('div', attrs={'class':'tweet-stats'}).find_all('span', attrs={'class':'tweet-stat'})
|
||||
for stat in stats:
|
||||
if 'comment' in str(stat):
|
||||
tweet['comments'] = stat.find('div',attrs={'class':'icon-container'}).text
|
||||
elif 'retweet' in str(stat):
|
||||
tweet['retweets'] = stat.find('div',attrs={'class':'icon-container'}).text
|
||||
elif 'heart' in str(stat):
|
||||
tweet['likes'] = stat.find('div',attrs={'class':'icon-container'}).text
|
||||
else:
|
||||
tweet['quotes'] = stat.find('div',attrs={'class':'icon-container'}).text
|
||||
feedPosts.append(tweet)
|
||||
else:
|
||||
return {"emptyFeed": True}
|
||||
|
||||
Reference in New Issue
Block a user