diff --git a/app/routes.py b/app/routes.py index 3a30022..93816a8 100644 --- a/app/routes.py +++ b/app/routes.py @@ -1,6 +1,8 @@ from flask_login import login_user, logout_user, current_user, login_required from flask import render_template, flash, redirect, url_for, request from app.forms import LoginForm, RegistrationForm, EmptyForm, SearchForm +from requests_futures.sessions import FuturesSession +from concurrent.futures import as_completed from app.models import User, twitterPost from werkzeug.urls import url_parse from bs4 import BeautifulSoup @@ -18,14 +20,20 @@ nitterInstanceII = "https://nitter.mastodont.cat" @app.route('/index') @login_required def index(): + start_time = time.time() following = current_user.following_list() followed = current_user.followed.count() posts = [] avatarPath = "img/avatars/1.png" form = EmptyForm() - [posts.extend(getPosts(fwd.username)) for fwd in following] + posts.extend(getFeed(following)) posts.sort(key=lambda x: x.timeStamp, reverse=True) - return render_template('index.html', title='Home', posts=posts, avatar=avatarPath, followedCount=followed, form=form) + if not posts: + profilePic = avatarPath + else: + profilePic = posts[0].userProfilePic + print("--- {} seconds fetching feed---".format(time.time() - start_time)) + return render_template('index.html', title='Home', posts=posts, avatar=avatarPath, profilePic = profilePic, followedCount=followed, form=form) @app.route('/login', methods=['GET', 'POST']) def login(): @@ -194,7 +202,11 @@ def user(username): posts.extend(getPosts(username)) form = EmptyForm() user = User.query.filter_by(username=username).first() - return render_template('user.html', user=user, posts=posts, form=form) + if not posts: + profilePic = avatarPath + else: + profilePic = posts[0].userProfilePic + return render_template('user.html', user=user, posts=posts, profilePic = profilePic, form=form) def getTimeDiff(t): tweetTime = datetime.datetime(*t[:6]) @@ -210,11 +222,50 @@ def getTimeDiff(t): return timeString def isTwitterUser(username): - request = requests.get('https://nitter.net/{}'.format(username), timeout=1) + request = requests.get('https://nitter.net/{}/rss'.format(username), timeout=5) if request.status_code == 404: return False return True +def getFeed(urls): + avatarPath = "img/avatars/{}.png".format(str(random.randint(1,12))) + feedPosts = [] + with FuturesSession() as session: + futures = [session.get('https://nitter.net/{}/rss'.format(u.username)) for u in urls] + for future in as_completed(futures): + resp = future.result() + rssFeed=feedparser.parse(resp.content) + if rssFeed.entries != []: + for post in rssFeed.entries: + newPost = twitterPost() + newPost.username = rssFeed.feed.title.split("/")[1].replace(" ", "") + newPost.twitterName = rssFeed.feed.title.split("/")[0] + newPost.date = getTimeDiff(post.published_parsed) + newPost.timeStamp = datetime.datetime(*post.published_parsed[:6]) + newPost.op = post.author + try: + newPost.userProfilePic = rssFeed.channel.image.url + except: + newPost.profilePicture = "" + newPost.urlToPost = post.link + newPost.content = Markup(post.description) + + if "Pinned" in post.title.split(":")[0]: + newPost.isPinned = True + + if "RT by" in post.title: + newPost.isRT = True + newPost.profilePic = "" + else: + newPost.isRT = False + try: + newPost.profilePic = rssFeed.channel.image.url + except: + newPost.profilePic = avatarPath + feedPosts.append(newPost) + time.sleep(1) + return feedPosts + def getPosts(account): avatarPath = "img/avatars/{}.png".format(str(random.randint(1,12))) posts = [] diff --git a/requirements.txt b/requirements.txt index 06241ff..31d3b12 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,6 @@ +aiohttp==3.6.2 alembic==1.4.2 +async-timeout==3.0.1 attrs==19.3.0 beautifulsoup4==4.9.1 bs4==0.0.1 @@ -15,6 +17,9 @@ Flask-Migrate==2.5.3 Flask-SQLAlchemy==2.4.3 Flask-WTF==0.14.3 future==0.18.2 +gevent==20.6.2 +greenlet==0.4.16 +grequests==0.6.0 idna==2.10 itsdangerous==1.1.0 Jinja2==2.11.2 @@ -22,14 +27,19 @@ llvmlite==0.33.0 lxml==4.5.2 Mako==1.1.3 MarkupSafe==1.1.1 +multidict==4.7.6 numpy==1.19.0 python-dateutil==2.8.1 python-dotenv==0.14.0 python-editor==1.0.4 requests==2.24.0 +requests-futures==1.0.0 six==1.15.0 soupsieve==2.0.1 SQLAlchemy==1.3.18 urllib3==1.25.9 Werkzeug==1.0.1 WTForms==2.3.1 +yarl==1.4.2 +zope.event==4.4 +zope.interface==5.1.0