From 6892d41c52fbcf2859bde5806bc44a154048ea28 Mon Sep 17 00:00:00 2001 From: Wire Date: Tue, 14 Jul 2020 14:24:43 +0200 Subject: [PATCH] Efficiency improvements + Pinned posts. --- app/models.py | 4 +- app/routes.py | 114 +++++++++++++--------------------- app/templates/_post.html | 4 +- app/templates/_post_nort.html | 5 +- app/templates/base.html | 2 +- app/templates/index.html | 19 +++--- app/templates/user.html | 4 +- requirements.txt | 6 ++ 8 files changed, 73 insertions(+), 85 deletions(-) diff --git a/app/models.py b/app/models.py index 7f8b12a..62d99b4 100644 --- a/app/models.py +++ b/app/models.py @@ -56,14 +56,16 @@ def load_user(id): class twitterPost(): date = 0 username = 'Error' + twitterName = "Error Name" + isPinned = False op = 'Error' isRT = True urlToPost = "" validPost = True content = "El gato siguió a la liebre. Esto es un texto de ejemplo." profilePic = "url" - twitterName = "Error Name" timeStamp = "error" + userProfilePic = "1.png" class Post(db.Model): id = db.Column(db.Integer, primary_key=True) diff --git a/app/routes.py b/app/routes.py index 88e8490..3a30022 100644 --- a/app/routes.py +++ b/app/routes.py @@ -13,7 +13,6 @@ import requests nitterInstance = "https://nitter.net/" nitterInstanceII = "https://nitter.mastodont.cat" -print("[ATTENTION]: New version. You will need to install BeautifulSoup: 'pip install bs4'") @app.route('/') @app.route('/index') @@ -24,39 +23,9 @@ def index(): posts = [] avatarPath = "img/avatars/1.png" form = EmptyForm() - for fwd in following: - avatarPath = "img/avatars/{}.png".format(str(random.randint(1,12))) - - #Gather profile info. - rssFeed = feedparser.parse('{instance}{user}/rss'.format(instance=nitterInstance, user=fwd.username)) - twitterAt = rssFeed.feed.title.split("/")[1].replace(" ", "") - twitterName = rssFeed.feed.title.split("/")[0] - - #Gather posts - if rssFeed.entries != []: - for post in rssFeed.entries: - newPost = twitterPost() - - newPost.username = rssFeed.feed.title.split("/")[0] - newPost.date = getTimeDiff(post.published_parsed) - newPost.timeStamp = datetime.datetime(*post.published_parsed[:6]) - newPost.op = post.author - newPost.urlToPost = post.link - newPost.content = Markup(post.description) - - if "RT by" in post.title: - newPost.isRT = True - newPost.profilePic = "" - else: - newPost.isRT = False - try: - newPost.profilePic = rssFeed.channel.image.url - except: - newPost.profilePic = avatarPath - posts.append(newPost) - posts.sort(key=lambda x: x.timeStamp, reverse=True) - return render_template('index.html', title='Home', posts=posts, avatar=avatarPath, followedCount=followed, twitterAt=twitterAt, twitterName=twitterName, form=form) - + [posts.extend(getPosts(fwd.username)) for fwd in following] + posts.sort(key=lambda x: x.timeStamp, reverse=True) + return render_template('index.html', title='Home', posts=posts, avatar=avatarPath, followedCount=followed, form=form) @app.route('/login', methods=['GET', 'POST']) def login(): @@ -188,7 +157,7 @@ def search(): user = form.username.data if isTwitterUser(user): r = requests.get("{instance}search?f=users&q={usern}".format(instance=nitterInstance, usern=user)) - html = BeautifulSoup(str(r.content), features="lxml") + html = BeautifulSoup(str(r.content), "lxml") results = html.body.find_all('a', attrs={'class':'tweet-link'}) parsedResults = [s['href'].replace("/", "") for s in results] @@ -221,42 +190,11 @@ def user(username): elif not isTwitter and user is None: return redirect(url_for('notfound')) - #Gather profile info. - rssFeed = feedparser.parse('{instance}{user}/rss'.format(instance=nitterInstance,user=username)) - try: - profilePicture = rssFeed.channel.image.url - except: - profilePicture = "" - twitterAt = rssFeed.feed.title.split("/")[1].replace(" ", "") - twitterName = rssFeed.feed.title.split("/")[0] - - #Gather posts posts = [] - for post in rssFeed.entries: - newPost = twitterPost() - newPost.username = rssFeed.feed.title.split("/")[0] - newPost.date = getTimeDiff(post.published_parsed) - newPost.timeStamp = datetime.datetime(*post.published_parsed[:6]) - newPost.op = post.author - newPost.urlToPost = post.link - newPost.content = Markup(post.description) - - if "RT by" in post.title: - newPost.isRT = True - newPost.profilePic = "" - else: - newPost.isRT = False - try: - newPost.profilePic = rssFeed.channel.image.url - except: - newPost.profilePic = avatarPath - - #validPost = True - posts.append(newPost) - + posts.extend(getPosts(username)) form = EmptyForm() user = User.query.filter_by(username=username).first() - return render_template('user.html', user=user, posts=posts, form=form, profilePic=profilePicture, twitterAt=twitterAt, twitterName=twitterName) + return render_template('user.html', user=user, posts=posts, form=form) def getTimeDiff(t): tweetTime = datetime.datetime(*t[:6]) @@ -273,7 +211,43 @@ def getTimeDiff(t): def isTwitterUser(username): request = requests.get('https://nitter.net/{}'.format(username), timeout=1) - print("User {name} is {boo} twitter.".format(name=username, boo=request.status_code == 404)) if request.status_code == 404: return False - return True \ No newline at end of file + return True + +def getPosts(account): + avatarPath = "img/avatars/{}.png".format(str(random.randint(1,12))) + posts = [] + + #Gather profile info. + rssFeed = feedparser.parse('{instance}{user}/rss'.format(instance=nitterInstance, user=account)) + #Gather posts + if rssFeed.entries != []: + for post in rssFeed.entries: + newPost = twitterPost() + newPost.username = rssFeed.feed.title.split("/")[1].replace(" ", "") + newPost.twitterName = rssFeed.feed.title.split("/")[0] + newPost.date = getTimeDiff(post.published_parsed) + newPost.timeStamp = datetime.datetime(*post.published_parsed[:6]) + newPost.op = post.author + try: + newPost.userProfilePic = rssFeed.channel.image.url + except: + newPost.profilePicture = "" + newPost.urlToPost = post.link + newPost.content = Markup(post.description) + + if "Pinned" in post.title.split(":")[0]: + newPost.isPinned = True + + if "RT by" in post.title: + newPost.isRT = True + newPost.profilePic = "" + else: + newPost.isRT = False + try: + newPost.profilePic = rssFeed.channel.image.url + except: + newPost.profilePic = avatarPath + posts.append(newPost) + return posts \ No newline at end of file diff --git a/app/templates/_post.html b/app/templates/_post.html index 3a8250f..8b462df 100644 --- a/app/templates/_post.html +++ b/app/templates/_post.html @@ -6,10 +6,10 @@ -
{{ post.op }}
+
{{ post.op }}
{{post.date}} - {{twitterAt}} retwitted + {{post.username}} retwitted

{{post.content}}

diff --git a/app/templates/_post_nort.html b/app/templates/_post_nort.html index 9b9f3e2..062fc16 100644 --- a/app/templates/_post_nort.html +++ b/app/templates/_post_nort.html @@ -6,9 +6,12 @@
-
{{ post.op }}
+
{{ post.op }}
{{post.date}} + {% if post.isPinned %} + Pinned + {%endif%}

{{post.content}}

diff --git a/app/templates/base.html b/app/templates/base.html index 2d975f7..7d3d11f 100644 --- a/app/templates/base.html +++ b/app/templates/base.html @@ -28,7 +28,7 @@ {% with messages = get_flashed_messages() %} {% if messages %} -
+
Information
diff --git a/app/templates/index.html b/app/templates/index.html index d0a7e23..165bc42 100644 --- a/app/templates/index.html +++ b/app/templates/index.html @@ -9,12 +9,15 @@
- {% for post in posts %} - {% if post.isRT %} - {% include '_post.html' %} - {% else %} - {% include '_post_nort.html' %} - {% endif %} - {% endfor %} -
+ {% if posts %} + {% for post in posts %} + {% if post.isRT %} + {% include '_post.html' %} + {% else %} + {% include '_post_nort.html' %} + {% endif %} + {% endfor %} + {% else %} +
No posts
+ {% endif %} {% endblock %} diff --git a/app/templates/user.html b/app/templates/user.html index 5d9f022..1636326 100644 --- a/app/templates/user.html +++ b/app/templates/user.html @@ -4,9 +4,9 @@
- {{ twitterAt }} + {{ twitterAt }}
-
{{ twitterName }}
+
{{ posts[0].twitterName }}
diff --git a/requirements.txt b/requirements.txt index dc9ed3f..06241ff 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,9 +1,11 @@ alembic==1.4.2 +attrs==19.3.0 beautifulsoup4==4.9.1 bs4==0.0.1 certifi==2020.6.20 chardet==3.0.4 click==7.1.2 +defusedxml==0.6.0 dnspython==1.16.0 email-validator==1.1.1 feedparser==5.2.1 @@ -12,11 +14,15 @@ Flask-Login==0.5.0 Flask-Migrate==2.5.3 Flask-SQLAlchemy==2.4.3 Flask-WTF==0.14.3 +future==0.18.2 idna==2.10 itsdangerous==1.1.0 Jinja2==2.11.2 +llvmlite==0.33.0 +lxml==4.5.2 Mako==1.1.3 MarkupSafe==1.1.1 +numpy==1.19.0 python-dateutil==2.8.1 python-dotenv==0.14.0 python-editor==1.0.4