Big efficiency improvements.

app/routes.py

@@ -1,6 +1,8 @@
 from flask_login import login_user, logout_user, current_user, login_required
 from flask import render_template, flash, redirect, url_for, request
 from app.forms import LoginForm, RegistrationForm, EmptyForm, SearchForm
+from requests_futures.sessions import FuturesSession
+from concurrent.futures import as_completed
 from app.models import User, twitterPost
 from werkzeug.urls import url_parse
 from bs4 import BeautifulSoup
@@ -18,14 +20,20 @@ nitterInstanceII = "https://nitter.mastodont.cat"
 @app.route('/index')
 @login_required
 def index():
+    start_time = time.time()
     following = current_user.following_list()
     followed = current_user.followed.count()
     posts = []
     avatarPath = "img/avatars/1.png"
     form = EmptyForm()
-    [posts.extend(getPosts(fwd.username)) for fwd in following]
+    posts.extend(getFeed(following))
     posts.sort(key=lambda x: x.timeStamp, reverse=True)
-    return render_template('index.html', title='Home', posts=posts, avatar=avatarPath, followedCount=followed, form=form)
+    if not posts:
+        profilePic = avatarPath
+    else:
+        profilePic = posts[0].userProfilePic
+    print("--- {} seconds fetching feed---".format(time.time() - start_time))
+    return render_template('index.html', title='Home', posts=posts, avatar=avatarPath, profilePic = profilePic, followedCount=followed, form=form)
 
 @app.route('/login', methods=['GET', 'POST'])
 def login():
@@ -194,7 +202,11 @@ def user(username):
     posts.extend(getPosts(username))
     form = EmptyForm()
     user = User.query.filter_by(username=username).first()
-    return render_template('user.html', user=user, posts=posts, form=form)
+    if not posts:
+        profilePic = avatarPath
+    else:
+        profilePic = posts[0].userProfilePic
+    return render_template('user.html', user=user, posts=posts, profilePic = profilePic, form=form)
 
 def getTimeDiff(t):
     tweetTime = datetime.datetime(*t[:6])
@@ -210,11 +222,50 @@ def getTimeDiff(t):
     return timeString
 
 def isTwitterUser(username):
-    request = requests.get('https://nitter.net/{}'.format(username), timeout=1)
+    request = requests.get('https://nitter.net/{}/rss'.format(username), timeout=5)
     if request.status_code == 404:
         return False
     return True
 
+def getFeed(urls):
+    avatarPath = "img/avatars/{}.png".format(str(random.randint(1,12)))
+    feedPosts = []
+    with FuturesSession() as session:
+        futures = [session.get('https://nitter.net/{}/rss'.format(u.username)) for u in urls]
+        for future in as_completed(futures):
+            resp = future.result()
+            rssFeed=feedparser.parse(resp.content)
+            if rssFeed.entries != []:
+                    for post in rssFeed.entries:
+                        newPost = twitterPost()
+                        newPost.username = rssFeed.feed.title.split("/")[1].replace(" ", "")
+                        newPost.twitterName = rssFeed.feed.title.split("/")[0]
+                        newPost.date = getTimeDiff(post.published_parsed)
+                        newPost.timeStamp = datetime.datetime(*post.published_parsed[:6])
+                        newPost.op = post.author
+                        try:
+                            newPost.userProfilePic = rssFeed.channel.image.url
+                        except:
+                            newPost.profilePicture = ""
+                        newPost.urlToPost = post.link
+                        newPost.content = Markup(post.description)
+                        
+                        if "Pinned" in post.title.split(":")[0]:
+                            newPost.isPinned = True
+
+                        if "RT by" in post.title:
+                            newPost.isRT = True
+                            newPost.profilePic = ""
+                        else:
+                            newPost.isRT = False
+                            try:
+                                newPost.profilePic = rssFeed.channel.image.url
+                            except:
+                                newPost.profilePic = avatarPath
+                        feedPosts.append(newPost)
+    time.sleep(1)
+    return feedPosts
+
 def getPosts(account):
     avatarPath = "img/avatars/{}.png".format(str(random.randint(1,12)))
     posts = []

requirements.txt

@@ -1,4 +1,6 @@
+aiohttp==3.6.2
 alembic==1.4.2
+async-timeout==3.0.1
 attrs==19.3.0
 beautifulsoup4==4.9.1
 bs4==0.0.1
@@ -15,6 +17,9 @@ Flask-Migrate==2.5.3
 Flask-SQLAlchemy==2.4.3
 Flask-WTF==0.14.3
 future==0.18.2
+gevent==20.6.2
+greenlet==0.4.16
+grequests==0.6.0
 idna==2.10
 itsdangerous==1.1.0
 Jinja2==2.11.2
@@ -22,14 +27,19 @@ llvmlite==0.33.0
 lxml==4.5.2
 Mako==1.1.3
 MarkupSafe==1.1.1
+multidict==4.7.6
 numpy==1.19.0
 python-dateutil==2.8.1
 python-dotenv==0.14.0
 python-editor==1.0.4
 requests==2.24.0
+requests-futures==1.0.0
 six==1.15.0
 soupsieve==2.0.1
 SQLAlchemy==1.3.18
 urllib3==1.25.9
 Werkzeug==1.0.1
 WTForms==2.3.1
+yarl==1.4.2
+zope.event==4.4
+zope.interface==5.1.0