New version 2020.07.27

app/forms.py

@@ -14,6 +14,10 @@ class SearchForm(FlaskForm):
     username = StringField('Username')
     submit = SubmitField('Search')
 
+class ChannelForm(FlaskForm):
+    channelId = StringField('Channel ID')
+    submit = SubmitField('Follow')
+
 
 class RegistrationForm(FlaskForm):
     username = StringField('Username', validators=[DataRequired()])

app/models.py

@@ -8,6 +8,11 @@ followers = db.Table('followers',
     db.Column('followed_id', db.Integer, db.ForeignKey('user.id'))
 )
 
+channel_association = db.Table('channel_association',
+    db.Column('channel_id', db.String, db.ForeignKey('channel.id')),
+    db.Column('user_id', db.Integer, db.ForeignKey('user.id'))
+) # Association: CHANNEL --followed by--> [USERS]
+
 class User(UserMixin, db.Model):
     id = db.Column(db.Integer, primary_key=True)
     username = db.Column(db.String(64), index=True, unique=True)
@@ -42,6 +47,8 @@ class User(UserMixin, db.Model):
     def saved_posts(self):
         return Post.query.filter_by(user_id=self.id)
 
+    def youtube_following_list(self):
+        return self.youtubeFollowed.all()
         
     followed = db.relationship(
         'User', secondary=followers,
@@ -49,6 +56,11 @@ class User(UserMixin, db.Model):
         secondaryjoin=(followers.c.followed_id == id),
         backref=db.backref('followers', lazy='dynamic'), lazy='dynamic')
 
+    youtubeFollowed = db.relationship("invidiousFollow",
+        secondary=channel_association,
+        back_populates="followers",
+        lazy='dynamic')
+
 
 @login.user_loader
 def load_user(id):
@@ -68,6 +80,27 @@ class twitterPost():
     timeStamp = "error"
     userProfilePic = "1.png"
 
+class invidiousPost():
+    channelName = 'Error'
+    channelUrl = '#'
+    videoUrl = '#'
+    videoTitle = '#'
+    videoThumb = '#'
+    description = "LOREM IPSUM"
+    date = 'None'
+
+
+class invidiousFollow(db.Model):
+    __tablename__ = 'channel'
+    id = db.Column(db.Integer, primary_key=True)
+    channelId = db.Column(db.String(30), nullable=False, unique=True)
+    followers = db.relationship('User', 
+                                secondary=channel_association,
+                                back_populates="youtubeFollowed")
+    
+    def __repr__(self):
+        return '<invidiousFollow {}>'.format(self.channelId)
+
 class Post(db.Model):
     id = db.Column(db.Integer, primary_key=True)
     body = db.Column(db.String(140))

app/routes.py

@@ -1,9 +1,9 @@
+from flask import render_template, flash, redirect, url_for, request, send_from_directory
+from app.forms import LoginForm, RegistrationForm, EmptyForm, SearchForm, ChannelForm
+from app.models import User, twitterPost, invidiousPost, Post, invidiousFollow
 from flask_login import login_user, logout_user, current_user, login_required
-from flask import render_template, flash, redirect, url_for, request
-from app.forms import LoginForm, RegistrationForm, EmptyForm, SearchForm
 from requests_futures.sessions import FuturesSession
 from concurrent.futures import as_completed
-from app.models import User, twitterPost, Post
 from werkzeug.urls import url_parse
 from bs4 import BeautifulSoup
 from flask import Markup
@@ -12,6 +12,8 @@ import time, datetime
 import random, string
 import feedparser
 import requests
+import json
+import re
 
 nitterInstance = "https://nitter.net/"
 nitterInstanceII = "https://nitter.mastodont.cat"
@@ -20,7 +22,7 @@ nitterInstanceII = "https://nitter.mastodont.cat"
 @app.route('/index')
 @login_required
 def index():
-    start_time = time.time()
+    #start_time = time.time()
     following = current_user.following_list()
     followed = current_user.followed.count()
     posts = []
@@ -32,9 +34,10 @@ def index():
         profilePic = avatarPath
     else:
         profilePic = posts[0].userProfilePic
-    print("--- {} seconds fetching feed---".format(time.time() - start_time))
+    #print("--- {} seconds fetching feed---".format(time.time() - start_time))
     return render_template('index.html', title='Home', posts=posts, avatar=avatarPath, profilePic = profilePic, followedCount=followed, form=form)
 
+
 @app.route('/login', methods=['GET', 'POST'])
 def login():
     if current_user.is_authenticated:
@@ -58,6 +61,46 @@ def logout():
     logout_user()
     return redirect(url_for('index'))
 
+@app.route('/settings')
+@login_required
+def settings():
+    return render_template('settings.html')
+
+@app.route('/export')
+@login_required
+#Export data into a JSON file. Later you can import the data.
+def export():
+    a = exportData()
+    if a:
+        return send_from_directory('data_export.json', as_attachment=True)
+    else:
+        return redirect(url_for('/error/405'))
+
+def exportData():
+    twitterFollowing = current_user.following_list()
+    youtubeFollowing = current_user.youtube_following_list()
+    data = {}
+    data['twitter'] = []
+    data['youtube'] = []
+
+    for f in twitterFollowing:
+        data['twitter'].append({
+            'username': f.username
+        })
+    
+    for f in youtubeFollowing:
+        data['youtube'].append({
+            'channelId': f.channelId
+        })
+
+    try:
+        with open('app/data_export.json', 'w') as outfile:
+            json.dump(data, outfile)
+        return True
+    except:
+        return False
+
+    
 
 @app.route('/register', methods=['GET', 'POST'])
 def register():
@@ -77,6 +120,32 @@ def register():
             return redirect(url_for('login'))
     return render_template('register.html', title='Register', form=form)
 
+@app.route('/invidious', methods=['GET', 'POST'])
+@login_required
+def invidious():
+    form = ChannelForm()
+    if form.validate_on_submit():
+        channelId = form.channelId.data
+        if requests.get('https://invidio.us/feed/channel/{}'.format(channelId)).status_code == 200:
+            follow = invidiousFollow()
+            follow.channelId = channelId
+            follow.followers.append(current_user)
+            try:
+                db.session.add(follow)
+                db.session.commit()
+                flash("Added to list!")
+            except:
+                flash("Something went wrong. Try again!")
+                return redirect(url_for('invidious'))
+        else:
+            flash("Enter a valid Channel ID. Eg: UCJWCJCWOxBYSi5DhCieLOLQ")
+            return redirect(url_for('invidious'))
+    ids = current_user.youtube_following_list()
+    videos = getInvidiousPosts(ids)
+    if videos:
+        videos.sort(key=lambda x: x.date, reverse=True)
+    return render_template('invidious.html', videos=videos, form=form)
+
 @app.route('/savePost/<url>', methods=['POST'])
 @login_required
 def savePost(url):
@@ -120,7 +189,7 @@ def follow(username):
         user = User.query.filter_by(username=username).first()
         isTwitter = isTwitterUser(username)
         if user is None and isTwitter:
-            x = ''.join(random.choice(string.ascii_uppercase + string.ascii_lowercase + string.digits) for _ in range(16))
+            x = ''.join(randomrandom.choice(string.ascii_uppercase + string.ascii_lowercase + string.digits) for _ in range(16))
             newUser = User(username=username, email="{}@person.is".format(x))
             db.session.add(newUser)
             db.session.commit()
@@ -288,7 +357,6 @@ def getFeed(urls):
                             except:
                                 newPost.profilePic = avatarPath
                         feedPosts.append(newPost)
-    time.sleep(1)
     return feedPosts
 
 def getPosts(account):
@@ -326,4 +394,25 @@ def getPosts(account):
                 except:
                     newPost.profilePic = avatarPath
             posts.append(newPost)
-    return posts
+    return posts
+
+def getInvidiousPosts(ids):
+    videos = []
+    with FuturesSession() as session:
+        futures = [session.get('https://invidio.us/feed/channel/{}'.format(id.channelId)) for id in ids]
+        for future in as_completed(futures):
+            resp = future.result()
+            rssFeed=feedparser.parse(resp.content)
+            for vid in rssFeed.entries:
+                video = invidiousPost()
+                video.date = vid.published_parsed
+                video.timeStamp = getTimeDiff(vid.published_parsed)
+                video.channelName = vid.author_detail.name
+                video.channelUrl = vid.author_detail.href
+                video.videoUrl = vid.link
+                video.videoTitle = vid.title
+                video.videoThumb = vid.media_thumbnail[0]['url']
+                video.description = vid.summary.split('<p style="word-break:break-word;white-space:pre-wrap">')[1]
+                video.description = re.sub(r'^https?:\/\/.*[\r\n]*', '', video.description[0:120]+"...", flags=re.MULTILINE)
+                videos.append(video)
+    return videos

app/templates/_empty_feed.html

@@ -0,0 +1,9 @@
+<div style="margin-top: 2em;" class="ui one column centered grid">
+    <div style="margin: 1.5em;" class="ui row">
+        <img class="ui medium circular image" src="{{ url_for('static',filename='img/empty.png') }}">
+    </div>
+
+    <div style="margin: 1.5em;" class="ui row">
+        <h2 class="ui header">This feed is empty.</h2>
+    </div>
+</div>

app/templates/_video.html

@@ -0,0 +1,26 @@
+<div class="card">
+    <div class="image">
+        <img src="{{video.videoThumb}}">
+    </div>
+    <div class="content">
+        <a class="video-title" target="_blank" href="{{video.videoUrl}}">{{video.videoTitle}}</a>
+        <div class="meta">
+            <a href="{{video.channelUrl}}">{{video.channelName}}</a>
+        </div>
+        <div class="description">
+            {{video.description}}
+        </div>
+    </div>
+    <div class="extra content">
+        <span class="right floated">
+            <i class="clock icon"></i>
+            {{video.timeStamp}}
+        </span>
+        <span>
+        <a href="{{video.videoUrl}}">
+            <i class="eye icon"></i>
+            Open
+        </a>
+        </span>
+    </div>
+</div>

app/templates/base.html

@@ -6,23 +6,40 @@
         {% if title %}
         <title>{{ title }} - Parassiter</title>
         {% else %}
-        <title>Welcome to Parasitter</title>
+        <title>Parasitter</title>
         {% endif %}
         <link rel= "stylesheet" type= "text/css" href= "{{ url_for('static',filename='semantic/semantic.min.css') }}">
+
+        <style>
+            .twitter{
+                color: rgb(0, 166, 196) !important;
+            }
+
+            .youtube{
+                color: rgb(224, 32, 32) !important;
+            }
+
+            .video-title{
+                font-weight: bold;
+                font-size: 1.15em;
+            }
+        </style>
     </head>
     <body>
         <div class="ui stackable menu">
             <div class="item">
             <img src="{{ url_for('static',filename='img/logo.png') }}">
             </div>
-            <a href="{{ url_for('index') }}" class="item">Home</a>
+            <a href="{{ url_for('index') }}" class="twitter item">Twitter</a>
             {% if current_user.is_anonymous %}
             <a href="{{ url_for('login') }}" class="item">Login</a>
             {% else %}
-            <a href="{{ url_for('search') }}" class="item">Search</a>
+                <a href="{{ url_for('search') }}" class="twitter item">Search</a>
-            <a href="{{ url_for('following') }}" class="item">Following</a>
+                <a href="{{ url_for('following') }}" class="twitter item">Following</a>
-            <a href="{{ url_for('saved') }}" class="item">Saved</a>
+                <a href="{{ url_for('saved') }}" class="twitter item">Saved</a>
-            <a href="{{ url_for('logout') }}" class="item">Logout</a>
+                <a href="{{ url_for('invidious') }}" class="youtube item">Youtube</a>
+                <a href="{{ url_for('logout') }}" class="item">Logout</a>
+                <a href="{{ url_for('settings') }}" class="item"><i class="cog icon"></i></a>
             {% endif %}
         </div>
 

app/templates/invidious.html

@@ -0,0 +1,30 @@
+{% extends "base.html" %}
+
+{% block content %}
+  <br>
+  <div class="ui one column centered grid">
+    <form class="ui form" action="" method="post" novalidate>
+      {{ form.hidden_tag() }}
+      <p>
+          {{ form.channelId.label }}<br>
+          {{ form.channelId(size=32) }}<br>
+          {% for error in form.channelId.errors %}
+          <span style="color: red;">[{{ error }}]</span>
+          {% endfor %}
+      </p>
+      <p>{{ form.submit() }}</p>
+    </form>
+  </div>
+  <br>
+  <br>
+
+    {% if videos %}
+      <div class="ui centered cards">
+        {% for video in videos %}
+          {% include '_video.html' %}
+        {% endfor %}
+      </div>
+    {% else %}
+      {% include '_empty_feed.html' %}
+    {% endif %}
+{% endblock %}

app/templates/settings.html

@@ -0,0 +1,47 @@
+{% extends "base.html" %}
+
+{% block content %}
+<br>
+<div class="ui one column centered grid">
+    <h2 class="ui icon header">
+        <i class="settings icon"></i>
+        <div class="content">
+            Settings
+            <div class="sub header">Manage your settings.</div>
+        </div>
+    </h2>
+</div>
+<hr>
+<br>
+
+<div class="ui one column centered grid">
+    <div class="ui relaxed divided list">
+        <div class="item">
+            <i class="large download middle aligned icon"></i>
+            <div class="content">
+            <a href="{{ url_for('export') }}" class="header">Export Data</a>
+            <div class="description">Export data into JSON file</div>
+            </div>
+        </div>
+        <div class="item">
+            <i class="large moon middle aligned icon"></i>
+            <div class="content">
+                <div class="ui slider checkbox">
+                    <input type="checkbox" name="newsletter">
+                    <label>Dark mode</label>
+                </div>
+            </div>
+        </div>
+        <div class="item">
+            <i class="large weight middle aligned icon"></i>
+            <div class="content">
+                <div class="ui slider checkbox">
+                    <input type="checkbox" name="newsletter">
+                    <label>Disable images</label>
+                    <div class="description">Show links instead</div>
+                </div>
+            </div>
+        </div>
+    </div>
+</div>
+{% endblock %}

app/templates/user.html

@@ -6,7 +6,7 @@
             <div class="center aligned author">
                 <img class="ui avatar image" src="{{ posts[0].userProfilePic }}"> {{ twitterAt }}
             </div>
-            <div style="margin: .1em" class="center aligned header"><a href="https://nitter.net/{{ posts[0].op.replace('@','') }}/">{{ posts[0].twitterName }}</a></div>
+            <div style="margin: .1em" class="center aligned header"><a href="https://nitter.net/{{ user.username }}">{{ posts[0].twitterName }}</a></div>
         <div class="center aligned description">
             <a>
                 <i class="users icon"></i>
@@ -40,15 +40,7 @@
     <div class="ui one column grid" id="card-container">
     
     {% if not posts %}
-        <div style="margin-top: 2em;" class="ui one column centered grid">
+        {% include '_empty_feed.html' %}
-            <div style="margin: 1.5em;" class="ui row">
-                <img class="ui medium circular image" src="{{ url_for('static',filename='img/empty.png') }}">
-            </div>
-        
-            <div style="margin: 1.5em;" class="ui row">
-                <h2 class="ui header">This feed is empty.</h2>
-            </div>
-        </div>
     {% else %}
         {% for post in posts %}
             {% if post.isRT %}

requirements.txt

@@ -2,6 +2,7 @@ alembic==1.4.2
 async-timeout==3.0.1
 attrs==19.3.0
 beautifulsoup4==4.9.1
+bleach==3.1.5
 bs4==0.0.1
 certifi==2020.6.20
 chardet==3.0.4
@@ -27,7 +28,9 @@ Mako==1.1.3
 MarkupSafe==1.1.1
 multidict==4.7.6
 numpy==1.19.0
+packaging==20.4
 PyMySQL==0.9.3
+pyparsing==2.4.7
 python-dateutil==2.8.1
 python-dotenv==0.14.0
 python-editor==1.0.4
@@ -37,6 +40,7 @@ six==1.15.0
 soupsieve==2.0.1
 SQLAlchemy==1.3.18
 urllib3==1.25.9
+webencodings==0.5.1
 Werkzeug==1.0.1
 WTForms==2.3.1
 yarl==1.4.2