New version 2020.07.27

2020-07-27 15:00:01 +02:00 · 2020-07-27 15:00:01 +02:00 · 4574581495
commit 4574581495
parent cbc244cdf0
10 changed files with 275 additions and 24 deletions
--- a/app/forms.py
+++ b/app/forms.py
@ -14,6 +14,10 @@ class SearchForm(FlaskForm):
    username = StringField('Username')
    submit = SubmitField('Search')

+class ChannelForm(FlaskForm):
+    channelId = StringField('Channel ID')
+    submit = SubmitField('Follow')
+

 class RegistrationForm(FlaskForm):
    username = StringField('Username', validators=[DataRequired()])
--- a/app/models.py
+++ b/app/models.py
@ -8,6 +8,11 @@ followers = db.Table('followers',
    db.Column('followed_id', db.Integer, db.ForeignKey('user.id'))
 )

+channel_association = db.Table('channel_association',
+    db.Column('channel_id', db.String, db.ForeignKey('channel.id')),
+    db.Column('user_id', db.Integer, db.ForeignKey('user.id'))
+) # Association: CHANNEL --followed by--> [USERS]
+
 class User(UserMixin, db.Model):
    id = db.Column(db.Integer, primary_key=True)
    username = db.Column(db.String(64), index=True, unique=True)
@ -42,6 +47,8 @@ class User(UserMixin, db.Model):
    def saved_posts(self):
        return Post.query.filter_by(user_id=self.id)

+    def youtube_following_list(self):
+        return self.youtubeFollowed.all()
        
    followed = db.relationship(
        'User', secondary=followers,
@ -49,6 +56,11 @@ class User(UserMixin, db.Model):
        secondaryjoin=(followers.c.followed_id == id),
        backref=db.backref('followers', lazy='dynamic'), lazy='dynamic')

+    youtubeFollowed = db.relationship("invidiousFollow",
+        secondary=channel_association,
+        back_populates="followers",
+        lazy='dynamic')
+

@login.user_loader
 def load_user(id):
@ -68,6 +80,27 @@ class twitterPost():
    timeStamp = "error"
    userProfilePic = "1.png"

+class invidiousPost():
+    channelName = 'Error'
+    channelUrl = '#'
+    videoUrl = '#'
+    videoTitle = '#'
+    videoThumb = '#'
+    description = "LOREM IPSUM"
+    date = 'None'
+
+
+class invidiousFollow(db.Model):
+    __tablename__ = 'channel'
+    id = db.Column(db.Integer, primary_key=True)
+    channelId = db.Column(db.String(30), nullable=False, unique=True)
+    followers = db.relationship('User', 
+                                secondary=channel_association,
+                                back_populates="youtubeFollowed")
+    
+    def __repr__(self):
+        return '<invidiousFollow {}>'.format(self.channelId)
+
 class Post(db.Model):
    id = db.Column(db.Integer, primary_key=True)
    body = db.Column(db.String(140))
--- a/app/routes.py
+++ b/app/routes.py
@ -1,9 +1,9 @@
+from flask import render_template, flash, redirect, url_for, request, send_from_directory
+from app.forms import LoginForm, RegistrationForm, EmptyForm, SearchForm, ChannelForm
+from app.models import User, twitterPost, invidiousPost, Post, invidiousFollow
 from flask_login import login_user, logout_user, current_user, login_required
-from flask import render_template, flash, redirect, url_for, request
-from app.forms import LoginForm, RegistrationForm, EmptyForm, SearchForm
 from requests_futures.sessions import FuturesSession
 from concurrent.futures import as_completed
-from app.models import User, twitterPost, Post
 from werkzeug.urls import url_parse
 from bs4 import BeautifulSoup
 from flask import Markup
@ -12,6 +12,8 @@ import time, datetime
 import random, string
 import feedparser
 import requests
+import json
+import re

 nitterInstance = "https://nitter.net/"
 nitterInstanceII = "https://nitter.mastodont.cat"
@ -20,7 +22,7 @@ nitterInstanceII = "https://nitter.mastodont.cat"
@app.route('/index')
@login_required
 def index():
-    start_time = time.time()
+    #start_time = time.time()
    following = current_user.following_list()
    followed = current_user.followed.count()
    posts = []
@ -32,9 +34,10 @@ def index():
        profilePic = avatarPath
    else:
        profilePic = posts[0].userProfilePic
-    print("--- {} seconds fetching feed---".format(time.time() - start_time))
+    #print("--- {} seconds fetching feed---".format(time.time() - start_time))
    return render_template('index.html', title='Home', posts=posts, avatar=avatarPath, profilePic = profilePic, followedCount=followed, form=form)

+
@app.route('/login', methods=['GET', 'POST'])
 def login():
    if current_user.is_authenticated:
@ -58,6 +61,46 @@ def logout():
    logout_user()
    return redirect(url_for('index'))

+@app.route('/settings')
+@login_required
+def settings():
+    return render_template('settings.html')
+
+@app.route('/export')
+@login_required
+#Export data into a JSON file. Later you can import the data.
+def export():
+    a = exportData()
+    if a:
+        return send_from_directory('data_export.json', as_attachment=True)
+    else:
+        return redirect(url_for('/error/405'))
+
+def exportData():
+    twitterFollowing = current_user.following_list()
+    youtubeFollowing = current_user.youtube_following_list()
+    data = {}
+    data['twitter'] = []
+    data['youtube'] = []
+
+    for f in twitterFollowing:
+        data['twitter'].append({
+            'username': f.username
+        })
+    
+    for f in youtubeFollowing:
+        data['youtube'].append({
+            'channelId': f.channelId
+        })
+
+    try:
+        with open('app/data_export.json', 'w') as outfile:
+            json.dump(data, outfile)
+        return True
+    except:
+        return False
+
+    

@app.route('/register', methods=['GET', 'POST'])
 def register():
@ -77,6 +120,32 @@ def register():
            return redirect(url_for('login'))
    return render_template('register.html', title='Register', form=form)

+@app.route('/invidious', methods=['GET', 'POST'])
+@login_required
+def invidious():
+    form = ChannelForm()
+    if form.validate_on_submit():
+        channelId = form.channelId.data
+        if requests.get('https://invidio.us/feed/channel/{}'.format(channelId)).status_code == 200:
+            follow = invidiousFollow()
+            follow.channelId = channelId
+            follow.followers.append(current_user)
+            try:
+                db.session.add(follow)
+                db.session.commit()
+                flash("Added to list!")
+            except:
+                flash("Something went wrong. Try again!")
+                return redirect(url_for('invidious'))
+        else:
+            flash("Enter a valid Channel ID. Eg: UCJWCJCWOxBYSi5DhCieLOLQ")
+            return redirect(url_for('invidious'))
+    ids = current_user.youtube_following_list()
+    videos = getInvidiousPosts(ids)
+    if videos:
+        videos.sort(key=lambda x: x.date, reverse=True)
+    return render_template('invidious.html', videos=videos, form=form)
+
@app.route('/savePost/<url>', methods=['POST'])
@login_required
 def savePost(url):
@ -120,7 +189,7 @@ def follow(username):
        user = User.query.filter_by(username=username).first()
        isTwitter = isTwitterUser(username)
        if user is None and isTwitter:
-            x = ''.join(random.choice(string.ascii_uppercase + string.ascii_lowercase + string.digits) for _ in range(16))
+            x = ''.join(randomrandom.choice(string.ascii_uppercase + string.ascii_lowercase + string.digits) for _ in range(16))
            newUser = User(username=username, email="{}@person.is".format(x))
            db.session.add(newUser)
            db.session.commit()
@ -288,7 +357,6 @@ def getFeed(urls):
                            except:
                                newPost.profilePic = avatarPath
                        feedPosts.append(newPost)
-    time.sleep(1)
    return feedPosts

 def getPosts(account):
@ -326,4 +394,25 @@ def getPosts(account):
                except:
                    newPost.profilePic = avatarPath
            posts.append(newPost)
-    return posts
+    return posts
+
+def getInvidiousPosts(ids):
+    videos = []
+    with FuturesSession() as session:
+        futures = [session.get('https://invidio.us/feed/channel/{}'.format(id.channelId)) for id in ids]
+        for future in as_completed(futures):
+            resp = future.result()
+            rssFeed=feedparser.parse(resp.content)
+            for vid in rssFeed.entries:
+                video = invidiousPost()
+                video.date = vid.published_parsed
+                video.timeStamp = getTimeDiff(vid.published_parsed)
+                video.channelName = vid.author_detail.name
+                video.channelUrl = vid.author_detail.href
+                video.videoUrl = vid.link
+                video.videoTitle = vid.title
+                video.videoThumb = vid.media_thumbnail[0]['url']
+                video.description = vid.summary.split('<p style="word-break:break-word;white-space:pre-wrap">')[1]
+                video.description = re.sub(r'^https?:\/\/.*[\r\n]*', '', video.description[0:120]+"...", flags=re.MULTILINE)
+                videos.append(video)
+    return videos
--- a/app/templates/_empty_feed.html
+++ b/app/templates/_empty_feed.html
@ -0,0 +1,9 @@
+<div style="margin-top: 2em;" class="ui one column centered grid">
+    <div style="margin: 1.5em;" class="ui row">
+        <img class="ui medium circular image" src="{{ url_for('static',filename='img/empty.png') }}">
+    </div>
+
+    <div style="margin: 1.5em;" class="ui row">
+        <h2 class="ui header">This feed is empty.</h2>
+    </div>
+</div>
--- a/app/templates/_video.html
+++ b/app/templates/_video.html
@ -0,0 +1,26 @@
+<div class="card">
+    <div class="image">
+        <img src="{{video.videoThumb}}">
+    </div>
+    <div class="content">
+        <a class="video-title" target="_blank" href="{{video.videoUrl}}">{{video.videoTitle}}</a>
+        <div class="meta">
+            <a href="{{video.channelUrl}}">{{video.channelName}}</a>
+        </div>
+        <div class="description">
+            {{video.description}}
+        </div>
+    </div>
+    <div class="extra content">
+        <span class="right floated">
+            <i class="clock icon"></i>
+            {{video.timeStamp}}
+        </span>
+        <span>
+        <a href="{{video.videoUrl}}">
+            <i class="eye icon"></i>
+            Open
+        </a>
+        </span>
+    </div>
+</div>
--- a/app/templates/base.html
+++ b/app/templates/base.html
@ -6,23 +6,40 @@
        {% if title %}
        <title>{{ title }} - Parassiter</title>
        {% else %}
-        <title>Welcome to Parasitter</title>
+        <title>Parasitter</title>
        {% endif %}
        <link rel= "stylesheet" type= "text/css" href= "{{ url_for('static',filename='semantic/semantic.min.css') }}">
+
+        <style>
+            .twitter{
+                color: rgb(0, 166, 196) !important;
+            }
+
+            .youtube{
+                color: rgb(224, 32, 32) !important;
+            }
+
+            .video-title{
+                font-weight: bold;
+                font-size: 1.15em;
+            }
+        </style>
    </head>
    <body>
        <div class="ui stackable menu">
            <div class="item">
            <img src="{{ url_for('static',filename='img/logo.png') }}">
            </div>
-            <a href="{{ url_for('index') }}" class="item">Home</a>
+            <a href="{{ url_for('index') }}" class="twitter item">Twitter</a>
            {% if current_user.is_anonymous %}
            <a href="{{ url_for('login') }}" class="item">Login</a>
            {% else %}
-            <a href="{{ url_for('search') }}" class="item">Search</a>
-            <a href="{{ url_for('following') }}" class="item">Following</a>
-            <a href="{{ url_for('saved') }}" class="item">Saved</a>
-            <a href="{{ url_for('logout') }}" class="item">Logout</a>
+                <a href="{{ url_for('search') }}" class="twitter item">Search</a>
+                <a href="{{ url_for('following') }}" class="twitter item">Following</a>
+                <a href="{{ url_for('saved') }}" class="twitter item">Saved</a>
+                <a href="{{ url_for('invidious') }}" class="youtube item">Youtube</a>
+                <a href="{{ url_for('logout') }}" class="item">Logout</a>
+                <a href="{{ url_for('settings') }}" class="item"><i class="cog icon"></i></a>
            {% endif %}
        </div>

--- a/app/templates/invidious.html
+++ b/app/templates/invidious.html
@ -0,0 +1,30 @@
+{% extends "base.html" %}
+
+{% block content %}
+  <br>
+  <div class="ui one column centered grid">
+    <form class="ui form" action="" method="post" novalidate>
+      {{ form.hidden_tag() }}
+      <p>
+          {{ form.channelId.label }}<br>
+          {{ form.channelId(size=32) }}<br>
+          {% for error in form.channelId.errors %}
+          <span style="color: red;">[{{ error }}]</span>
+          {% endfor %}
+      </p>
+      <p>{{ form.submit() }}</p>
+    </form>
+  </div>
+  <br>
+  <br>
+
+    {% if videos %}
+      <div class="ui centered cards">
+        {% for video in videos %}
+          {% include '_video.html' %}
+        {% endfor %}
+      </div>
+    {% else %}
+      {% include '_empty_feed.html' %}
+    {% endif %}
+{% endblock %}
--- a/app/templates/settings.html
+++ b/app/templates/settings.html
@ -0,0 +1,47 @@
+{% extends "base.html" %}
+
+{% block content %}
+<br>
+<div class="ui one column centered grid">
+    <h2 class="ui icon header">
+        <i class="settings icon"></i>
+        <div class="content">
+            Settings
+            <div class="sub header">Manage your settings.</div>
+        </div>
+    </h2>
+</div>
+<hr>
+<br>
+
+<div class="ui one column centered grid">
+    <div class="ui relaxed divided list">
+        <div class="item">
+            <i class="large download middle aligned icon"></i>
+            <div class="content">
+            <a href="{{ url_for('export') }}" class="header">Export Data</a>
+            <div class="description">Export data into JSON file</div>
+            </div>
+        </div>
+        <div class="item">
+            <i class="large moon middle aligned icon"></i>
+            <div class="content">
+                <div class="ui slider checkbox">
+                    <input type="checkbox" name="newsletter">
+                    <label>Dark mode</label>
+                </div>
+            </div>
+        </div>
+        <div class="item">
+            <i class="large weight middle aligned icon"></i>
+            <div class="content">
+                <div class="ui slider checkbox">
+                    <input type="checkbox" name="newsletter">
+                    <label>Disable images</label>
+                    <div class="description">Show links instead</div>
+                </div>
+            </div>
+        </div>
+    </div>
+</div>
+{% endblock %}
--- a/app/templates/user.html
+++ b/app/templates/user.html
@ -6,7 +6,7 @@
            <div class="center aligned author">
                <img class="ui avatar image" src="{{ posts[0].userProfilePic }}"> {{ twitterAt }}
            </div>
-            <div style="margin: .1em" class="center aligned header"><a href="https://nitter.net/{{ posts[0].op.replace('@','') }}/">{{ posts[0].twitterName }}</a></div>
+            <div style="margin: .1em" class="center aligned header"><a href="https://nitter.net/{{ user.username }}">{{ posts[0].twitterName }}</a></div>
        <div class="center aligned description">
            <a>
                <i class="users icon"></i>
@ -40,15 +40,7 @@
    <div class="ui one column grid" id="card-container">
    
    {% if not posts %}
-        <div style="margin-top: 2em;" class="ui one column centered grid">
-            <div style="margin: 1.5em;" class="ui row">
-                <img class="ui medium circular image" src="{{ url_for('static',filename='img/empty.png') }}">
-            </div>
-        
-            <div style="margin: 1.5em;" class="ui row">
-                <h2 class="ui header">This feed is empty.</h2>
-            </div>
-        </div>
+        {% include '_empty_feed.html' %}
    {% else %}
        {% for post in posts %}
            {% if post.isRT %}
--- a/requirements.txt
+++ b/requirements.txt
@ -2,6 +2,7 @@ alembic==1.4.2
 async-timeout==3.0.1
 attrs==19.3.0
 beautifulsoup4==4.9.1
+bleach==3.1.5
 bs4==0.0.1
 certifi==2020.6.20
 chardet==3.0.4
@ -27,7 +28,9 @@ Mako==1.1.3
 MarkupSafe==1.1.1
 multidict==4.7.6
 numpy==1.19.0
+packaging==20.4
 PyMySQL==0.9.3
+pyparsing==2.4.7
 python-dateutil==2.8.1
 python-dotenv==0.14.0
 python-editor==1.0.4
@ -37,6 +40,7 @@ six==1.15.0
 soupsieve==2.0.1
 SQLAlchemy==1.3.18
 urllib3==1.25.9
+webencodings==0.5.1
 Werkzeug==1.0.1
 WTForms==2.3.1
 yarl==1.4.2