From 4574581495a6d57214a808d4eb6f3dfebb94184b Mon Sep 17 00:00:00 2001 From: pluja Date: Mon, 27 Jul 2020 15:00:01 +0200 Subject: [PATCH] New version 2020.07.27 --- app/forms.py | 4 ++ app/models.py | 33 +++++++++++ app/routes.py | 105 ++++++++++++++++++++++++++++++--- app/templates/_empty_feed.html | 9 +++ app/templates/_video.html | 26 ++++++++ app/templates/base.html | 29 +++++++-- app/templates/invidious.html | 30 ++++++++++ app/templates/settings.html | 47 +++++++++++++++ app/templates/user.html | 12 +--- requirements.txt | 4 ++ 10 files changed, 275 insertions(+), 24 deletions(-) create mode 100644 app/templates/_empty_feed.html create mode 100644 app/templates/_video.html create mode 100644 app/templates/invidious.html create mode 100644 app/templates/settings.html diff --git a/app/forms.py b/app/forms.py index adb4feb..158a481 100644 --- a/app/forms.py +++ b/app/forms.py @@ -14,6 +14,10 @@ class SearchForm(FlaskForm): username = StringField('Username') submit = SubmitField('Search') +class ChannelForm(FlaskForm): + channelId = StringField('Channel ID') + submit = SubmitField('Follow') + class RegistrationForm(FlaskForm): username = StringField('Username', validators=[DataRequired()]) diff --git a/app/models.py b/app/models.py index be3bb81..82c71da 100644 --- a/app/models.py +++ b/app/models.py @@ -8,6 +8,11 @@ followers = db.Table('followers', db.Column('followed_id', db.Integer, db.ForeignKey('user.id')) ) +channel_association = db.Table('channel_association', + db.Column('channel_id', db.String, db.ForeignKey('channel.id')), + db.Column('user_id', db.Integer, db.ForeignKey('user.id')) +) # Association: CHANNEL --followed by--> [USERS] + class User(UserMixin, db.Model): id = db.Column(db.Integer, primary_key=True) username = db.Column(db.String(64), index=True, unique=True) @@ -42,6 +47,8 @@ class User(UserMixin, db.Model): def saved_posts(self): return Post.query.filter_by(user_id=self.id) + def youtube_following_list(self): + return self.youtubeFollowed.all() followed = db.relationship( 'User', secondary=followers, @@ -49,6 +56,11 @@ class User(UserMixin, db.Model): secondaryjoin=(followers.c.followed_id == id), backref=db.backref('followers', lazy='dynamic'), lazy='dynamic') + youtubeFollowed = db.relationship("invidiousFollow", + secondary=channel_association, + back_populates="followers", + lazy='dynamic') + @login.user_loader def load_user(id): @@ -68,6 +80,27 @@ class twitterPost(): timeStamp = "error" userProfilePic = "1.png" +class invidiousPost(): + channelName = 'Error' + channelUrl = '#' + videoUrl = '#' + videoTitle = '#' + videoThumb = '#' + description = "LOREM IPSUM" + date = 'None' + + +class invidiousFollow(db.Model): + __tablename__ = 'channel' + id = db.Column(db.Integer, primary_key=True) + channelId = db.Column(db.String(30), nullable=False, unique=True) + followers = db.relationship('User', + secondary=channel_association, + back_populates="youtubeFollowed") + + def __repr__(self): + return ''.format(self.channelId) + class Post(db.Model): id = db.Column(db.Integer, primary_key=True) body = db.Column(db.String(140)) diff --git a/app/routes.py b/app/routes.py index ed48aa2..b1b00a2 100644 --- a/app/routes.py +++ b/app/routes.py @@ -1,9 +1,9 @@ +from flask import render_template, flash, redirect, url_for, request, send_from_directory +from app.forms import LoginForm, RegistrationForm, EmptyForm, SearchForm, ChannelForm +from app.models import User, twitterPost, invidiousPost, Post, invidiousFollow from flask_login import login_user, logout_user, current_user, login_required -from flask import render_template, flash, redirect, url_for, request -from app.forms import LoginForm, RegistrationForm, EmptyForm, SearchForm from requests_futures.sessions import FuturesSession from concurrent.futures import as_completed -from app.models import User, twitterPost, Post from werkzeug.urls import url_parse from bs4 import BeautifulSoup from flask import Markup @@ -12,6 +12,8 @@ import time, datetime import random, string import feedparser import requests +import json +import re nitterInstance = "https://nitter.net/" nitterInstanceII = "https://nitter.mastodont.cat" @@ -20,7 +22,7 @@ nitterInstanceII = "https://nitter.mastodont.cat" @app.route('/index') @login_required def index(): - start_time = time.time() + #start_time = time.time() following = current_user.following_list() followed = current_user.followed.count() posts = [] @@ -32,9 +34,10 @@ def index(): profilePic = avatarPath else: profilePic = posts[0].userProfilePic - print("--- {} seconds fetching feed---".format(time.time() - start_time)) + #print("--- {} seconds fetching feed---".format(time.time() - start_time)) return render_template('index.html', title='Home', posts=posts, avatar=avatarPath, profilePic = profilePic, followedCount=followed, form=form) + @app.route('/login', methods=['GET', 'POST']) def login(): if current_user.is_authenticated: @@ -58,6 +61,46 @@ def logout(): logout_user() return redirect(url_for('index')) +@app.route('/settings') +@login_required +def settings(): + return render_template('settings.html') + +@app.route('/export') +@login_required +#Export data into a JSON file. Later you can import the data. +def export(): + a = exportData() + if a: + return send_from_directory('data_export.json', as_attachment=True) + else: + return redirect(url_for('/error/405')) + +def exportData(): + twitterFollowing = current_user.following_list() + youtubeFollowing = current_user.youtube_following_list() + data = {} + data['twitter'] = [] + data['youtube'] = [] + + for f in twitterFollowing: + data['twitter'].append({ + 'username': f.username + }) + + for f in youtubeFollowing: + data['youtube'].append({ + 'channelId': f.channelId + }) + + try: + with open('app/data_export.json', 'w') as outfile: + json.dump(data, outfile) + return True + except: + return False + + @app.route('/register', methods=['GET', 'POST']) def register(): @@ -77,6 +120,32 @@ def register(): return redirect(url_for('login')) return render_template('register.html', title='Register', form=form) +@app.route('/invidious', methods=['GET', 'POST']) +@login_required +def invidious(): + form = ChannelForm() + if form.validate_on_submit(): + channelId = form.channelId.data + if requests.get('https://invidio.us/feed/channel/{}'.format(channelId)).status_code == 200: + follow = invidiousFollow() + follow.channelId = channelId + follow.followers.append(current_user) + try: + db.session.add(follow) + db.session.commit() + flash("Added to list!") + except: + flash("Something went wrong. Try again!") + return redirect(url_for('invidious')) + else: + flash("Enter a valid Channel ID. Eg: UCJWCJCWOxBYSi5DhCieLOLQ") + return redirect(url_for('invidious')) + ids = current_user.youtube_following_list() + videos = getInvidiousPosts(ids) + if videos: + videos.sort(key=lambda x: x.date, reverse=True) + return render_template('invidious.html', videos=videos, form=form) + @app.route('/savePost/', methods=['POST']) @login_required def savePost(url): @@ -120,7 +189,7 @@ def follow(username): user = User.query.filter_by(username=username).first() isTwitter = isTwitterUser(username) if user is None and isTwitter: - x = ''.join(random.choice(string.ascii_uppercase + string.ascii_lowercase + string.digits) for _ in range(16)) + x = ''.join(randomrandom.choice(string.ascii_uppercase + string.ascii_lowercase + string.digits) for _ in range(16)) newUser = User(username=username, email="{}@person.is".format(x)) db.session.add(newUser) db.session.commit() @@ -288,7 +357,6 @@ def getFeed(urls): except: newPost.profilePic = avatarPath feedPosts.append(newPost) - time.sleep(1) return feedPosts def getPosts(account): @@ -326,4 +394,25 @@ def getPosts(account): except: newPost.profilePic = avatarPath posts.append(newPost) - return posts \ No newline at end of file + return posts + +def getInvidiousPosts(ids): + videos = [] + with FuturesSession() as session: + futures = [session.get('https://invidio.us/feed/channel/{}'.format(id.channelId)) for id in ids] + for future in as_completed(futures): + resp = future.result() + rssFeed=feedparser.parse(resp.content) + for vid in rssFeed.entries: + video = invidiousPost() + video.date = vid.published_parsed + video.timeStamp = getTimeDiff(vid.published_parsed) + video.channelName = vid.author_detail.name + video.channelUrl = vid.author_detail.href + video.videoUrl = vid.link + video.videoTitle = vid.title + video.videoThumb = vid.media_thumbnail[0]['url'] + video.description = vid.summary.split('

')[1] + video.description = re.sub(r'^https?:\/\/.*[\r\n]*', '', video.description[0:120]+"...", flags=re.MULTILINE) + videos.append(video) + return videos \ No newline at end of file diff --git a/app/templates/_empty_feed.html b/app/templates/_empty_feed.html new file mode 100644 index 0000000..b5861b0 --- /dev/null +++ b/app/templates/_empty_feed.html @@ -0,0 +1,9 @@ +

+
+ +
+ +
+

This feed is empty.

+
+
\ No newline at end of file diff --git a/app/templates/_video.html b/app/templates/_video.html new file mode 100644 index 0000000..1ecfd64 --- /dev/null +++ b/app/templates/_video.html @@ -0,0 +1,26 @@ +
+
+ +
+
+ {{video.videoTitle}} +
+ {{video.channelName}} +
+
+ {{video.description}} +
+
+
+ + + {{video.timeStamp}} + + + + + Open + + +
+
\ No newline at end of file diff --git a/app/templates/base.html b/app/templates/base.html index d31ba27..822c776 100644 --- a/app/templates/base.html +++ b/app/templates/base.html @@ -6,23 +6,40 @@ {% if title %} {{ title }} - Parassiter {% else %} - Welcome to Parasitter + Parasitter {% endif %} + +
- Home + {% if current_user.is_anonymous %} Login {% else %} - Search - Following - Saved - Logout + + + + Youtube + Logout + {% endif %}
diff --git a/app/templates/invidious.html b/app/templates/invidious.html new file mode 100644 index 0000000..f6026e6 --- /dev/null +++ b/app/templates/invidious.html @@ -0,0 +1,30 @@ +{% extends "base.html" %} + +{% block content %} +
+
+
+ {{ form.hidden_tag() }} +

+ {{ form.channelId.label }}
+ {{ form.channelId(size=32) }}
+ {% for error in form.channelId.errors %} + [{{ error }}] + {% endfor %} +

+

{{ form.submit() }}

+
+
+
+
+ + {% if videos %} +
+ {% for video in videos %} + {% include '_video.html' %} + {% endfor %} +
+ {% else %} + {% include '_empty_feed.html' %} + {% endif %} +{% endblock %} \ No newline at end of file diff --git a/app/templates/settings.html b/app/templates/settings.html new file mode 100644 index 0000000..2caa132 --- /dev/null +++ b/app/templates/settings.html @@ -0,0 +1,47 @@ +{% extends "base.html" %} + +{% block content %} +
+
+

+ +
+ Settings +
Manage your settings.
+
+

+
+
+
+ +
+
+
+ +
+ Export Data +
Export data into JSON file
+
+
+
+ +
+
+ + +
+
+
+
+ +
+
+ + +
Show links instead
+
+
+
+
+
+{% endblock %} \ No newline at end of file diff --git a/app/templates/user.html b/app/templates/user.html index 1636326..e46b4a1 100644 --- a/app/templates/user.html +++ b/app/templates/user.html @@ -6,7 +6,7 @@
{{ twitterAt }}
-
{{ posts[0].twitterName }}
+
{{ posts[0].twitterName }}
@@ -40,15 +40,7 @@
{% if not posts %} -
-
- -
- -
-

This feed is empty.

-
-
+ {% include '_empty_feed.html' %} {% else %} {% for post in posts %} {% if post.isRT %} diff --git a/requirements.txt b/requirements.txt index f5c9d1f..a0184a5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,6 +2,7 @@ alembic==1.4.2 async-timeout==3.0.1 attrs==19.3.0 beautifulsoup4==4.9.1 +bleach==3.1.5 bs4==0.0.1 certifi==2020.6.20 chardet==3.0.4 @@ -27,7 +28,9 @@ Mako==1.1.3 MarkupSafe==1.1.1 multidict==4.7.6 numpy==1.19.0 +packaging==20.4 PyMySQL==0.9.3 +pyparsing==2.4.7 python-dateutil==2.8.1 python-dotenv==0.14.0 python-editor==1.0.4 @@ -37,6 +40,7 @@ six==1.15.0 soupsieve==2.0.1 SQLAlchemy==1.3.18 urllib3==1.25.9 +webencodings==0.5.1 Werkzeug==1.0.1 WTForms==2.3.1 yarl==1.4.2