Update README.md

I have no idea how this even got here, git blame shows that this was done when the Nginx image was merged.

.dockerignore

@@ -8,3 +8,4 @@ docker-compose.yml
 LICENSE
 *.md
 dockerhash.txt
+app/static

.github/workflows/docker-build.yml

@@ -11,20 +11,20 @@ jobs:
   cpython-build-docker:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v2.3.4
         with:
           fetch-depth: 0
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
+        uses: docker/setup-qemu-action@v1.2.0
         with:
           platforms: all
       - name: Set up Docker Buildx
         id: buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@v1.5.1
         with:
           version: latest
       - name: Login to DockerHub
-        uses: docker/login-action@v1
+        uses: docker/login-action@v1.10.0
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
@@ -33,14 +33,14 @@ jobs:
       - name: Write the current version to a file
         run: "{ git describe --tags --abbrev=0 & date +\"%d-%m-%y\" & git rev-list HEAD --max-count=1 --abbrev-commit;} > version.txt"
       - name: cache docker cache
-        uses: actions/cache@v2.1.3
+        uses: actions/cache@v2.1.6
         with:
           path: ${{ github.workspace }}/cache
           key: ${{ runner.os }}-docker-cpython-${{ hashFiles('**/requirements.txt') }}-${{ hashFiles('**/dockerhash.txt') }}
           restore-keys: |
             ${{ runner.os }}-docker-cpython-
       - name: Build and push
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@v2.6.1
         with:
           context: .
           file: ./Dockerfile
@@ -52,20 +52,20 @@ jobs:
   pypy-build-docker:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v2.3.4
         with:
           fetch-depth: 0
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
+        uses: docker/setup-qemu-action@v1.2.0
         with:
           platforms: all
       - name: Set up Docker Buildx
         id: buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@v1.5.1
         with:
           version: latest
       - name: Login to DockerHub
-        uses: docker/login-action@v1
+        uses: docker/login-action@v1.10.0
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
@@ -74,14 +74,14 @@ jobs:
       - name: Write the current version to a file
         run: "{ git describe --tags --abbrev=0 & date +\"%d-%m-%y\" & git rev-list HEAD --max-count=1 --abbrev-commit;} > version.txt"
       - name: cache docker cache
-        uses: actions/cache@v2.1.3
+        uses: actions/cache@v2.1.6
         with:
           path: ${{ github.workspace }}/cache
           key: ${{ runner.os }}-docker-pypy-${{ hashFiles('**/requirements.txt') }}-${{ hashFiles('**/dockerhash.txt') }}
           restore-keys: |
             ${{ runner.os }}-docker-pypy-
       - name: Build and push
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@v2.6.1
         with:
           context: .
           file: ./pypy.Dockerfile
@@ -90,3 +90,44 @@ jobs:
           tags: ytorg/yotter:pypy
           cache-from: type=local,src=cache
           cache-to: type=local,dest=cache
+  nginx-build-docker:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2.3.4
+        with:
+          fetch-depth: 0
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v1.2.0
+        with:
+          platforms: all
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v1.5.1
+        with:
+          version: latest
+      - name: Login to DockerHub
+        uses: docker/login-action@v1.10.0
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+      - name: Get hash of latest image
+        run: docker pull nginx:mainline-alpine && docker inspect --format='{{index .RepoDigests 0}}' nginx:mainline-alpine > dockerhash.txt
+      - name: Write the current version to a file
+        run: "{ git describe --tags --abbrev=0 & date +\"%d-%m-%y\" & git rev-list HEAD --max-count=1 --abbrev-commit;} > version.txt"
+      - name: cache docker cache
+        uses: actions/cache@v2.1.6
+        with:
+          path: ${{ github.workspace }}/cache
+          key: ${{ runner.os }}-docker-nginx-${{ hashFiles('**/dockerhash.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-docker-nginx-
+      - name: Build and push
+        uses: docker/build-push-action@v2.6.1
+        with:
+          context: .
+          file: ./Dockerfile
+          platforms: linux/amd64,linux/arm64
+          push: true
+          tags: ytorg/nginx:latest
+          cache-from: type=local,src=cache
+          cache-to: type=local,dest=cache

Dockerfile

@@ -8,7 +8,7 @@ WORKDIR /usr/src/app
 COPY ./requirements.txt /usr/src/app
 
 # Build Dependencies
-RUN apk --no-cache add gcc musl-dev libffi-dev openssl-dev libxml2-dev libxslt-dev file llvm-dev make g++
+RUN apk --no-cache add gcc musl-dev libffi-dev openssl-dev libxml2-dev libxslt-dev file llvm-dev make g++ cargo rust
 
 # Python Dependencies
 RUN pip install --no-cache-dir --prefix=/install wheel cryptography gunicorn pymysql

README.md

@@ -1,8 +1,12 @@
+## This project is no longer maintained. Visit [this repo](https://github.com/TeamPiped/Piped) for an alternative.
+
 <p align="center"> <img width="700" src="app/static/img/banner.png"> </img></p> 
 <p align="center">
    <a href="https://www.gnu.org/licenses/gpl-3.0"><img alt="License: GPL v3" src="https://img.shields.io/badge/License-AGPLv3-blue.svg"></img></a>
    <a href="https://github.com/pluja/Yotter"><img alt="Development state" src="https://img.shields.io/badge/State-Beta-blue.svg"></img></a>
    <a href="https://github.com/pluja/Yotter/pulls"><img alt="Pull Requests Welcome" src="https://img.shields.io/badge/PRs-Welcome-green.svg"></img></a>
+   <a href="https://git.kavin.rocks/kavin/Yotter"><img alt="Mirror 1" src="https://img.shields.io/badge/Mirror1-git.kavin.rocks-teal"></img></a>
+   <a href="https://84.38.177.154/libremirrors/ytorg/Yotter"><img alt="Mirror 2" src="https://img.shields.io/badge/Mirror2-git.rip-purple"></img></a>
 </p> 
 
 Yotter allows you to follow and gather all the content from your favorite Twitter and YouTube accounts in a *beautiful* feed so you can stay up to date without compromising your privacy at all. Yotter is written with Python and Flask and uses Semantic-UI as its CSS framework.
@@ -12,6 +16,7 @@ Yotter is possible thanks to several open-source projects that are listed on the
 # Index:
 * [Why](#why)
 * [Features](#features)
+   * [Roadmap](#roadmap)
 * [FAQ](#FAQ)
 * [Privacy and Security](#privacy)
 * [Public instances](#public-instances)
@@ -29,7 +34,7 @@ With the *particular* data about you, they can get money from the highest bidder
 
 Further more, they don't care about **what you in particular watch**, this is only sold to the highest bidder who then may or may not do the harm. What they care more about is **what people watch** this is the important data and the one that allows to manipulate, bias, censor, etc.
 
-So we need platforms and spaces where we can freely watch and listen content without these watchful eyes upon us. Ideally, everyone would use a free (as in freedom) and decentralized platform like [Peertube](https://joinpeertube.org/), [LBRY](https://lbry.tv/), [Mastodon](https://joinmastodon.org/) or [Pleroma](https://pleroma.social/) but things are not like this. The main multimedia content factory is Youtube and the microblogging king is Twitter. So we will do whatever is possible to be able to watch and read the content and avoid the surveillance that seeks us these days. We will resist.
+So we need platforms and spaces where we can freely watch and listen content without these watchful eyes upon us. Ideally, everyone would use a free (as in freedom) and decentralized platform like [Peertube](https://joinpeertube.org/), [Odysee](https://odysee.com/), [Mastodon](https://joinmastodon.org/) or [Pleroma](https://pleroma.social/) but things are not like this. The main multimedia content factory is Youtube and the microblogging king is Twitter. So we will do whatever is possible to be able to watch and read the content and avoid the surveillance that seeks us these days. We will resist.
 
 # Features:
 - [x] No Ads.
@@ -48,11 +53,24 @@ So we need platforms and spaces where we can freely watch and listen content wit
 
 *Video player is VideoJS, which uses JavaScript. But if JavaScript is disabled Yotter still works perfectly and uses the default HTML video player.
 
+### Roadmap
+The following features are planned to be implemented in the near future:
+* [ ] Improve performance and efficiency
+
+#### Youtube specific:
+* [ ] Subtitles
+* [ ] > 720p Quality
+* [ ] Save youtube videos
+* [ ] Support for live streams
+
+#### Twitter specific:
+* [ ] Translations
+
 # FAQ
 ### What's the difference between this and Invidious?
-At first I started working on this project as a solution for following Twitter accounts (a thing that can't be done with Nitter) and getting a Twitter-like feed. Weeks later the leader of Invidious, Omar Roth, announced that he was stepping away from the project. As an Invidious active user, this made me think that a new alternative was needed for the community and also an alternative with an easier programmin language for most people (as Invidious is written in Crystal). So I started developing a '*written-in-python Invidious alternative*' and it went quite well.
+At first I started working on this project as a solution for following Twitter accounts (a thing that can't be done with Nitter) and getting a Twitter-like feed. Weeks later the leader of Invidious, Omar Roth, announced that he was stepping away from the project. As an Invidious active user, this made me think that a new alternative was needed for the community and also an alternative with an easier programming language for most people (as Invidious is written in Crystal). So I started developing a '*written-in-python Invidious alternative*' and it went quite well.
 
-I hope that this project can prosperate, gain contributors, new instances and create a good community around it.
+I hope that this project can prosper, gain contributors, new instances and create a good community around it.
 
 ### Why do I have to register to use Yotter?
 
@@ -66,7 +84,7 @@ Admins are allowed to remove restrictions on any page they want. [Check this sec
 
 If you want to use Yotter, it is recommended to self-host your own instance. You can use it for personal use or open it to the world. Self-hosting makes Yotter stronger and gives you full power. See [self hosting guide](https://github.com/ytorg/Yotter/blob/dev-indep/SELF-HOSTING.md).
 
-### Will you ever implement video recomendations, trendig videos, etc?
+### Will you ever implement video recommendations, trending videos, etc?
 No. From my point of view, these are toxic features. I, and every user, should be using all *social media* to get the content **they** want. Recomendations, trending, autoplay next video, etc. are all features designed to trap users on using the app, to make them forget about the time spent there and to create an addiction to it. No, I won't implement any toxic features on Yotter. Yotter will keep the UI clean, fast and simple. 
 
 You get your feed from followed accounts and you can search for any video you like. Only thing I would consider implementing would be some kind of page where you can ask for recommendations for a particular video. This way the user would, voluntarily, ask for the recommendations rather than having a temptation to click on a new, youtube-bias-recommended video.
@@ -129,24 +147,14 @@ These are projects that either make Yotter possible as an **essential part** of
 * [Video.js](https://videojs.com/)
 * [Invidious](https://github.com/iv-org/invidious)
 
-# Donate
+# [Donate](https://github.com/pluja/pluja/blob/main/SUPPORT.md)
+
+[Click here to see donation options](https://github.com/pluja/pluja/blob/main/SUPPORT.md)
+
 This project is completely free and Open Source and will always be.
 
 Donations are used to mantain the [yotter.xyz](https://yotter.xyz/) public instance. [This is the server](https://www.netcup.eu/bestellen/produkt.php?produkt=2598) that I have rented for now.
 
-#### Crypto:
-##### Preferred
-- **Bitcoin**: `bc1q5y3g907ju0pt40me7dr9fy5uhfhfgfv9k3kh3z`
-- **Monero**: `48nQGAXaC6eFK2Wo7SVVyF9xL333gDHjzdmRL3XETEqbU3w4CcKjjHVUZPU4W3dg1oJL8be3iGtUAQsgV88dzbS7QNpZjC2`
-
-##### Others:
-- **Ethereum**: `0x6cf0B1C3354c255F1a876f4833A1BBE82D887Ad6`
-- **Litecoin**: `MHjnpYHwu4y4AeQvVBDv52T8V6BzVxmiNZ`
-- **ZCash**: `t1a6smU9a6dxGfZWcX9bCRzE31qsaF27FsD`
-
-#### Fiat:
-- <a href="https://liberapay.com/pluja/donate"><img alt="Donate using Liberapay" src="https://liberapay.com/assets/widgets/donate.svg"></a>
-
 ## Screenshots
 #### Twitter / Tweets / Profiles
 <p align="center"> <img width="720" src="https://i.imgur.com/tA15ciH.png"> </img></p> 

SELF-HOSTING.md

@@ -123,7 +123,7 @@ If after the MySQL-server installation you have not been prompted to create a pa
 * `pip install cryptography`
 * `pip install -r requirements.txt`
 
-> You can edit the `yotter-config.json` file. [Check out all the options here](https://github.com/ytorg/Yotter/blob/dev-indep/README.md#configure-the-server)
+> You can edit the `yotter-config.json` file. [Check out all the options here](#configure-the-server)
 
 5. Install gunicorn (production web server for Python apps) and pymysql:
 `pip install gunicorn pymysql`
@@ -203,9 +203,69 @@ killasgroup=true
 After you write this configuration file, you have to reload the supervisor service for it to be imported:
 `sudo supervisorctl reload`
 
-#### Step 4: Nginx set up and HTTPS
+#### Step 4: Set up Nginx, http3 proxy and HTTPS
 The Yotter application server powered by gunicorn is now running privately port 8000. Now we need to expose the application to the outside world by enabling public facing web server on ports 80 and 443, the two ports too need to be opened on the firewall to handle the web traffic of the application. I want this to be a secure deployment, so I'm going to configure port 80 to forward all traffic to port 443, which is going to be encrypted. [ref](https://blog.miguelgrinberg.com/post/the-flask-mega-tutorial-part-xvii-deployment-on-linux).
 
+First we will get and set up the `http3-ytproxy`. For this we will need to [install go](https://github.com/golang/go/wiki/Ubuntu) but if you are on Ubuntu 20.04 or you have `snap` installed you can just run `sudo snap install --classic go` to get `go` installed.
+
+Then you will need to run the following commands:
+```
+cd $HOME
+git clone https://github.com/FireMasterK/http3-ytproxy
+cd http3-ytproxy
+go build -ldflags "-s -w" main.go
+mv main http3-ytproxy
+mkdir socket
+chown -R www-data:www-data socket
+```
+
+Now we will configure a `systemd` service to run the http3-ytproxy. For this you will need to `sudo nano /lib/systemd/system/http3-ytproxy.service` to start a the `nano` text editor. Now copy and paste this and save:
+
+> IMPORTANT: You may need to change some paths to fit your system!
+
+```
+[Unit]
+Description=Sleep service
+ConditionPathExists=/home/ubuntu/http3-ytproxy/http3-ytproxy
+After=network.target
+ 
+[Service]
+Type=simple
+User=www-data
+Group=www-data
+LimitNOFILE=1024
+
+Restart=on-failure
+RestartSec=10
+
+WorkingDirectory=/home/ubuntu/http3-ytproxy
+ExecStart=/home/ubuntu/http3-ytproxy/http3-ytproxy
+
+# make sure log directory exists and owned by syslog
+PermissionsStartOnly=true
+ExecStartPre=/bin/mkdir -p /var/log/http3-ytproxy
+ExecStartPre=/bin/chown syslog:adm /var/log/http3-ytproxy
+ExecStartPre=/bin/chmod 755 /var/log/http3-ytproxy
+StandardOutput=syslog
+StandardError=syslog
+SyslogIdentifier=http3-ytproxy
+ 
+[Install]
+WantedBy=multi-user.target
+```
+
+> IMPORTANT NOTE: Some distros have the Nginx user as `nginx` instead of `www-data`, if this is the case you should change the `User=` and `Group=` variables from the service file.
+
+Now you are ready to enable and start the service:
+```
+sudo systemctl enable http3-ytproxy.service
+sudo systemctl start http3-ytproxy.service
+```
+
+If you did everything ok you should see no errors when running `sudo journalctl -f -u http3-ytproxy`.
+
+Now we will set up Nginx. To do so:
+
 * `sudo rm /etc/nginx/sites-enabled/default`
 
 Create a new Nginx site, you can run `sudo nano /etc/nginx/sites-enabled/yotter`
@@ -225,15 +285,21 @@ server {
         expires 30d;
     }
     
-    location ~ (/videoplayback|/vi/|/a/) {
-       proxy_buffering off;
-       resolver 1.1.1.1;
-       proxy_pass https://$arg_host;
-       proxy_set_header Host $arg_host;
-       add_header Access-Control-Allow-Origin *;
-     }
+    location ~ (^/videoplayback$|/videoplayback/|/vi/|/a/|/ytc/) {
+        proxy_pass http://unix:/home/ubuntu/http3-ytproxy/socket/http-proxy.sock;
+        add_header Access-Control-Allow-Origin *;
+        sendfile on;
+        tcp_nopush on;
+        aio_write on;
+        aio threads=default;
+        directio 512;
+        proxy_http_version 1.1;
+        proxy_set_header Connection "";
+    }
 }
 ```
+> Note: You may need to change the proxy-pass line to fit your system. It should point to the socket created on the `http3-ytproxy/socket` folder.
+
 Make sure to replace `<yourdomain>` by the domain you are willing to use for your instance (i.e example.com). You can now edit `yotter-config.json` and set `isInstance` to `true`.
 
 You will also need to change the `</path/to>` after `alias` to fit your system. You have to point to the Yotter folder, in this set up it would be `/home/ubuntu` as it is the location where we cloned the Yotter app. This alias is created to handle static files directly, without forwarding to the application.
@@ -251,6 +317,8 @@ Now we will run certbot and we need to tell that we run an nginx server. Here yo
 
 [Follow this instructions to install certbot and generate an ssl certificate so your server can use HTTPS](https://certbot.eff.org/lets-encrypt/ubuntufocal-nginx)
 
+Finally, once this is done, you should edit the `yotter` nginx config and change the `listen 443 ssl;` line to `listen 443 ssl http2;`
+
 #### Updating the server
 Updating the server should always be pretty easy. These steps need to be run on the Yotter folder and with the python virtual env activated.
 

app/models.py

@@ -27,11 +27,11 @@ class User(UserMixin, db.Model):
     posts = db.relationship('Post', backref='author', lazy='dynamic')
 
     def __repr__(self):
-        return '<User {}>'.format(self.username)
+        return f'<User {self.username}>'
 
     def set_last_seen(self):
         self.last_seen = datetime.utcnow()
-    
+
     def set_admin_user(self):
         self.is_admin = True
 
@@ -40,7 +40,7 @@ class User(UserMixin, db.Model):
 
     def check_password(self, password):
         return check_password_hash(self.password_hash, password)
-    
+
     def follow(self, user):
         if not self.is_following(user):
             self.followed.append(user)
@@ -52,7 +52,7 @@ class User(UserMixin, db.Model):
     def is_following(self, user):
         return self.followed.filter(
             followers.c.followed_id == user.id).count() > 0
-    
+
     def following_list(self):
         return self.followed.all()
 
@@ -62,7 +62,7 @@ class User(UserMixin, db.Model):
     # TWITTER
     def twitter_following_list(self):
         return self.twitterFollowed.all()
-    
+
     def is_following_tw(self, uname):
         temp_cid = twitterFollow.query.filter_by(username = uname).first()
         if temp_cid is None:
@@ -73,11 +73,11 @@ class User(UserMixin, db.Model):
                 if f.username == uname:
                     return True
         return False
-    
+
     # YOUTUBE
     def youtube_following_list(self):
         return self.youtubeFollowed.all()
-    
+
     def is_following_yt(self, cid):
         temp_cid = youtubeFollow.query.filter_by(channelId = cid).first()
         if temp_cid is None:
@@ -88,7 +88,7 @@ class User(UserMixin, db.Model):
                 if f.channelId == cid:
                     return True
         return False
-        
+
     followed = db.relationship(
         'User', secondary=followers,
         primaryjoin=(followers.c.follower_id == id),
@@ -148,23 +148,23 @@ class youtubeFollow(db.Model):
     id = db.Column(db.Integer, primary_key=True)
     channelId = db.Column(db.String(30), nullable=False)
     channelName = db.Column(db.String(100))
-    followers = db.relationship('User', 
+    followers = db.relationship('User',
                                 secondary=channel_association,
                                 back_populates="youtubeFollowed")
-    
+
     def __repr__(self):
-        return '<youtubeFollow {}>'.format(self.channelName)
+        return f'<youtubeFollow {self.channelName}>'
 
 class twitterFollow(db.Model):
     __tablename__ = 'twitterAccount'
     id = db.Column(db.Integer, primary_key=True)
     username = db.Column(db.String(100), nullable=False)
-    followers = db.relationship('User', 
+    followers = db.relationship('User',
                                 secondary=twitter_association,
                                 back_populates="twitterFollowed")
-    
+
     def __repr__(self):
-        return '<twitterFollow {}>'.format(self.username)
+        return f'<twitterFollow {self.username}>'
 
 class Post(db.Model):
     id = db.Column(db.Integer, primary_key=True)
@@ -175,5 +175,4 @@ class Post(db.Model):
     user_id = db.Column(db.Integer, db.ForeignKey('user.id'))
 
     def __repr__(self):
-        return '<Post {}>'.format(self.body)
-        
+        return f'<Post {self.body}>'

app/routes.py

@@ -91,32 +91,32 @@ def twitter(page=0):
         followList.append(f.username)
     posts = []
 
-    cache_file = glob.glob("app/cache/{}_*".format(current_user.username))
+    cache_file = glob.glob(f"app/cache/{current_user.username}_*")
     if (len(cache_file) > 0):
         time_diff = round(time.time() - os.path.getmtime(cache_file[0]))
     else:
         time_diff = 999
 
-    # If cache file is more than 1 minute old
-    if page == 0 and time_diff > 60:
+    # If cache file is older than 30 minute old
+    if page == 0 and time_diff > 30:
         if cache_file:
             for f in cache_file:
                 os.remove(f)
         feed = nitterfeed.get_feed(followList)
-        cache_file = "{u}_{d}.json".format(u=current_user.username, d=time.strftime("%Y%m%d-%H%M%S"))
-        with open("app/cache/{}".format(cache_file), 'w') as fp:
+        cache_file = f"{current_user.username}_{time.strftime('%Y%m%d-%H%M%S')}.json"
+        with open(f"app/cache/{cache_file}", 'w') as fp:
             json.dump(feed, fp)
 
     # Else, refresh feed
     else:
         try:
-            cache_file = glob.glob("app/cache/{}*".format(current_user.username))[0]
+            cache_file = glob.glob(f"app/cache/{current_user.username}*")[0]
             with open(cache_file, 'r') as fp:
                 feed = json.load(fp)
         except:
             feed = nitterfeed.get_feed(followList)
-            cache_file = "{u}_{d}.json".format(u=current_user.username, d=time.strftime("%Y%m%d-%H%M%S"))
-            with open("app/cache/{}".format(cache_file), 'w') as fp:
+            cache_file = f"{current_user.username}_{time.strftime('%Y%m%d-%H%M%S')}.json"
+            with open(f"app/cache/{cache_file}", 'w') as fp:
                 json.dump(feed, fp)
 
     posts.extend(feed)
@@ -187,7 +187,7 @@ def follow(username):
     form = EmptyForm()
     if form.validate_on_submit():
         if followTwitterAccount(username):
-            flash("{} followed!".format(username))
+            flash(f"{username} followed!")
     return redirect(request.referrer)
 
 
@@ -202,7 +202,7 @@ def followTwitterAccount(username):
                 db.session.commit()
                 return True
             except:
-                flash("Twitter: Couldn't follow {}. Already followed?".format(username))
+                flash(f"Twitter: Couldn't follow {username}. Already followed?")
                 return False
     else:
         flash("Something went wrong... try again")
@@ -215,7 +215,7 @@ def unfollow(username):
     form = EmptyForm()
     if form.validate_on_submit():
         if twUnfollow(username):
-            flash("{} unfollowed!".format(username))
+            flash(f"{username} unfollowed!")
     return redirect(request.referrer)
 
 
@@ -248,7 +248,7 @@ def search():
         if results:
             return render_template('search.html', form=form, results=results, config=config)
         else:
-            flash("User {} not found...".format(user))
+            flash(f"User {user} not found...")
             return redirect(request.referrer)
     else:
         return render_template('search.html', form=form, config=config)
@@ -262,7 +262,7 @@ def u(username, page=1):
     if username == "favicon.ico":
         return redirect(url_for('static', filename='favicons/favicon.ico'))
     form = EmptyForm()
-    avatarPath = "img/avatars/{}.png".format(str(random.randint(1, 12)))
+    avatarPath = f"img/avatars/{str(random.randint(1, 12))}.png"
     user = nitteruser.get_user_info(username)
     if not user:
         flash("This user is not on Twitter.")
@@ -281,7 +281,7 @@ def u(username, page=1):
         prev_page = 0
     else:
         prev_page = page-1
-    
+
     if page > 2:
         page =2
 
@@ -300,7 +300,7 @@ def youtube():
     videos = getYoutubePosts(ids)
     if videos:
         videos.sort(key=lambda x: x.date, reverse=True)
-    print("--- {} seconds fetching youtube feed---".format(time.time() - start_time))
+    print(f"--- {time.time() - start_time} seconds fetching youtube feed---")
     return render_template('youtube.html', title="Yotter | Youtube", videos=videos, followCount=followCount,
                            config=config)
 
@@ -337,22 +337,21 @@ def ytsearch():
         filters = {"time": 0, "type": 0, "duration": 0}
         results = yts.search_by_terms(query, page, autocorrect, sort, filters)
 
-        next_page = "/ytsearch?q={q}&s={s}&p={p}".format(q=query, s=sort, p=int(page) + 1)
+        next_page = f"/ytsearch?q={query}&s={sort}&p={int(page)+1}"
         if int(page) == 1:
-            prev_page = "/ytsearch?q={q}&s={s}&p={p}".format(q=query, s=sort, p=1)
+            prev_page = f"/ytsearch?q={query}&s={sort}&p={1}"
         else:
-            prev_page = "/ytsearch?q={q}&s={s}&p={p}".format(q=query, s=sort, p=int(page) - 1)
+            prev_page = f"/ytsearch?q={query}&s={sort}&p={int(page)-1}"
 
         for video in results['videos']:
             hostname = urllib.parse.urlparse(video['videoThumb']).netloc
-            video['videoThumb'] = video['videoThumb'].replace("https://{}".format(hostname), "") + "&host=" + hostname
+            video['videoThumb'] = video['videoThumb'].replace(f"https://{hostname}", "") + "&host=" + hostname
 
         for channel in results['channels']:
             if config['isInstance']:
                 channel['thumbnail'] = channel['thumbnail'].replace("~", "/")
                 hostName = urllib.parse.urlparse(channel['thumbnail']).netloc
-                channel['thumbnail'] = channel['thumbnail'].replace("https://{}".format(hostName),
-                                                                    "") + "?host=" + hostName
+                channel['thumbnail'] = channel['thumbnail'].replace(f"https://{hostName}", "") + "?host=" + hostName
         return render_template('ytsearch.html', form=form, btform=button_form, results=results,
                                restricted=config['restrictPublicUsage'], config=config, npage=next_page,
                                ppage=prev_page)
@@ -371,7 +370,7 @@ def followYoutubeChannel(channelId):
     try:
         try:
             if not current_user.is_following_yt(channelId):
-                channelData = ytch.get_channel_tab_info(channelId, tab='about')
+                channelData = ytch.get_channel_tab(channelId, tab='about')
                 if channelData == False:
                     return False
                 follow = youtubeFollow()
@@ -380,7 +379,7 @@ def followYoutubeChannel(channelId):
                 follow.followers.append(current_user)
                 db.session.add(follow)
                 db.session.commit()
-                flash("{} followed!".format(channelData['channel_name']))
+                flash(f"{channelData['channel_name']} followed!")
                 return True
             else:
                 return False
@@ -388,8 +387,8 @@ def followYoutubeChannel(channelId):
             print(e)
             return False
     except KeyError as ke:
-        print("KeyError: {}:'{}' could not be found".format(ke, channelId))
-        flash("Youtube: ChannelId '{}' is not valid".format(channelId))
+        print(f"KeyError: {ke}:'{channelId}' could not be found")
+        flash(f"Youtube: ChannelId '{channelId}' is not valid")
         return False
 
 
@@ -410,7 +409,7 @@ def unfollowYoutubeChannel(channelId):
         if channel:
             db.session.delete(channel)
             db.session.commit()
-        flash("{} unfollowed!".format(name))
+        flash(f"{name} unfollowed!")
     except:
         flash("There was an error unfollowing the user. Try again.")
 
@@ -430,27 +429,26 @@ def channel(id):
     if sort is None:
         sort = 3
 
-    data = ytch.get_channel_tab_info(id, page, sort)
-
+    data = ytch.get_channel_tab(id, page, sort)
     for video in data['items']:
         if config['isInstance']:
             hostName = urllib.parse.urlparse(video['thumbnail'][1:]).netloc
-            video['thumbnail'] = video['thumbnail'].replace("https://{}".format(hostName), "")[1:].replace("hqdefault",
-                                                                                                           "mqdefault") + "&host=" + hostName
+            video['thumbnail'] = video['thumbnail'].replace(f"https://{hostName}", "")[1:].replace("hqdefault",
+                                                                                                   "mqdefault") + "&host=" + hostName
         else:
             video['thumbnail'] = video['thumbnail'].replace('/', '~')
 
     if config['isInstance']:
         hostName = urllib.parse.urlparse(data['avatar'][1:]).netloc
-        data['avatar'] = data['avatar'].replace("https://{}".format(hostName), "")[1:] + "?host=" + hostName
+        data['avatar'] = data['avatar'].replace(f"https://{hostName}", "")[1:] + "?host=" + hostName
     else:
         data['avatar'] = data['avatar'].replace('/', '~')
 
-    next_page = "/channel/{q}?s={s}&p={p}".format(q=id, s=sort, p=int(page) + 1)
+    next_page = f"/channel/{id}?s={sort}&p={int(page)+1}"
     if int(page) == 1:
-        prev_page = "/channel/{q}?s={s}&p={p}".format(q=id, s=sort, p=1)
+        prev_page = f"/channel/{id}?s={sort}&p={1}"
     else:
-        prev_page = "/channel/{q}?s={s}&p={p}".format(q=id, s=sort, p=int(page) - 1)
+        prev_page = f"/channel/{id}?s={sort}&p={int(page)-1}"
 
     return render_template('channel.html', form=form, btform=button_form, data=data,
                            restricted=config['restrictPublicUsage'], config=config, next_page=next_page,
@@ -488,11 +486,11 @@ def watch():
     if info['error'] == False:
         for format in info['formats']:
             hostName = urllib.parse.urlparse(format['url']).netloc
-            format['url'] = format['url'].replace("https://{}".format(hostName), "") + "&host=" + hostName
+            format['url'] = format['url'].replace(f"https://{hostName}", "") + "&host=" + hostName
 
         for format in info['audio_formats']:
             hostName = urllib.parse.urlparse(format['url']).netloc
-            format['url'] = format['url'].replace("https://{}".format(hostName), "") + "&host=" + hostName
+            format['url'] = format['url'].replace(f"https://{hostName}", "") + "&host=" + hostName
 
         # Markup description
         try:
@@ -804,7 +802,7 @@ def status():
 
 @app.route('/error/<errno>')
 def error(errno):
-    return render_template('{}.html'.format(str(errno)), config=config)
+    return render_template(f'{str(errno)}.html', config=config)
 
 
 def getTimeDiff(t):
@@ -812,24 +810,26 @@ def getTimeDiff(t):
 
     if diff.days == 0:
         if diff.seconds > 3599:
-            timeString = "{}h".format(int((diff.seconds / 60) / 60))
+            num = int((diff.seconds / 60) / 60)
+            timeString = f"{num}h"
         else:
-            timeString = "{}m".format(int(diff.seconds / 60))
+            num = int(diff.seconds / 60)
+            timeString = f"{num}m"
     else:
-        timeString = "{}d".format(diff.days)
+        timeString = f"{diff.days}d"
     return timeString
 
 
 def isTwitterUser(username):
-    response = requests.get('{instance}{user}/rss'.format(instance=NITTERINSTANCE, user=username))
+    response = requests.get(f'{NITTERINSTANCE}{username}/rss')
     if response.status_code == 404:
         return False
     return True
 
 
 def twitterUserSearch(terms):
-    response = urllib.request.urlopen(
-        '{instance}search?f=users&q={user}'.format(instance=NITTERINSTANCE, user=urllib.parse.quote(terms))).read()
+    url = f'{NITTERINSTANCE}search?f=users&q={urllib.parse.quote(terms)}'
+    response = urllib.request.urlopen(url).read()
     html = BeautifulSoup(str(response), "lxml")
 
     results = []
@@ -843,14 +843,14 @@ def twitterUserSearch(terms):
                     'unicode_escape').encode('latin_1').decode('utf8'),
                 "username": item.find('a', attrs={'class': 'username'}).getText().encode('latin_1').decode(
                     'unicode_escape').encode('latin_1').decode('utf8'),
-                'avatar': "{i}{s}".format(i=NITTERINSTANCE, s=item.find('img', attrs={'class': 'avatar'})['src'][1:])
+                'avatar': NITTERINSTANCE + item.find('img', attrs={'class': 'avatar'})['src'][1:],
             }
             results.append(user)
         return results
 
 
 def getTwitterUserInfo(username):
-    response = urllib.request.urlopen('{instance}{user}'.format(instance=NITTERINSTANCE, user=username)).read()
+    response = urllib.request.urlopen('{NITTERINSTANCE}{username}').read()
     # rssFeed = feedparser.parse(response.content)
 
     html = BeautifulSoup(str(response), "lxml")
@@ -881,9 +881,7 @@ def getTwitterUserInfo(username):
             "followers": numerize.numerize(
                 int(html.find_all('span', attrs={'class': 'profile-stat-num'})[2].string.replace(",", ""))),
             "likes": html.find_all('span', attrs={'class': 'profile-stat-num'})[3].string,
-            "profilePic": "{instance}{pic}".format(instance=NITTERINSTANCE,
-                                                   pic=html.find('a', attrs={'class': 'profile-card-avatar'})['href'][
-                                                       1:])
+            "profilePic": NITTERINSTANCE + html.find('a', attrs={'class': 'profile-card-avatar'})['href'][1:],
         }
         return user
 
@@ -891,7 +889,7 @@ def getTwitterUserInfo(username):
 def getFeed(urls):
     feedPosts = []
     with FuturesSession() as session:
-        futures = [session.get('{instance}{user}'.format(instance=NITTERINSTANCE, user=u.username)) for u in urls]
+        futures = [session.get(f'{NITTERINSTANCE}{u.username}') for u in urls]
         for future in as_completed(futures):
             res= future.result().content
             html = BeautifulSoup(res, "html.parser")
@@ -960,7 +958,7 @@ def getPosts(account):
     feedPosts = []
 
     # Gather profile info.
-    rssFeed = urllib.request.urlopen('{instance}{user}'.format(instance=NITTERINSTANCE, user=account)).read()
+    rssFeed = urllib.request.urlopen(f'{NITTERINSTANCE}{account}').read()
     # Gather feedPosts
     res = rssFeed.decode('utf-8')
     html = BeautifulSoup(res, "html.parser")
@@ -1018,8 +1016,7 @@ def getPosts(account):
 def getYoutubePosts(ids):
     videos = []
     with FuturesSession() as session:
-        futures = [session.get('https://www.youtube.com/feeds/videos.xml?channel_id={id}'.format(id=id.channelId)) for
-                   id in ids]
+        futures = [session.get(f'https://www.youtube.com/feeds/videos.xml?channel_id={id.channelId}') for id in ids]
         for future in as_completed(futures):
             resp = future.result()
             rssFeed = feedparser.parse(resp.content)
@@ -1050,7 +1047,7 @@ def getYoutubePosts(ids):
                     video.timeStamp = getTimeDiff(vid.published_parsed)
                 except:
                     if time != 0:
-                        video.timeStamp = "{} days".format(str(time.days))
+                        video.timeStamp = f"{str(time.days)} days"
                     else:
                         video.timeStamp = "Unknown"
 
@@ -1061,7 +1058,7 @@ def getYoutubePosts(ids):
                 video.videoTitle = vid.title
                 if config['isInstance']:
                     hostName = urllib.parse.urlparse(vid.media_thumbnail[0]['url']).netloc
-                    video.videoThumb = vid.media_thumbnail[0]['url'].replace("https://{}".format(hostName), "").replace(
+                    video.videoThumb = vid.media_thumbnail[0]['url'].replace(f"https://{hostName}", "").replace(
                         "hqdefault", "mqdefault") + "?host=" + hostName
                 else:
                     video.videoThumb = vid.media_thumbnail[0]['url'].replace('/', '~')
@@ -1070,4 +1067,4 @@ def getYoutubePosts(ids):
                 video.description = re.sub(r'^https?:\/\/.*[\r\n]*', '', video.description[0:120] + "...",
                                            flags=re.MULTILINE)
                 videos.append(video)
-    return videos
+    return videos

app/templates/channel.html

@@ -32,14 +32,14 @@
         <form action="{{ url_for('ytfollow', channelId=data.channel_id) }}" method="post">
             <button type="submit" value="Submit" class="ui red button">
                 <i class="user icon"></i>
-                Suscribe
+                Subscribe
             </button>
         </form>
     {% else %}
         <form action="{{ url_for('ytunfollow', channelId=data.channel_id) }}" method="post">
             <button type="submit" value="Submit" class="ui red active button">
                 <i class="user icon"></i>
-                Unsuscribe
+                Unsubscribe
             </button>
         </form>
     {%endif%}
@@ -93,4 +93,4 @@
         <a href="{{next_page}}"> <button class="right attached ui button"><i class="angle red right icon"></i></button></a>
     </div>
     <br>
-{% endblock %}
+{% endblock %}

app/templates/home.html

@@ -41,9 +41,9 @@
     <div class="text container ui">
         <div class="ui warning message">
             <div class="header">
-                {{config.admin_message_title}}
+                {{config.admin_message_title|safe}}
             </div>
-                {{config.admin_message}}
+                {{config.admin_message|safe}}
         </div>
     </div>
 {% endif %}

app/templates/search.html

@@ -6,7 +6,7 @@
         {{ form.hidden_tag() }}
         <p>
             {{ form.username.label }}<br>
-            {{ form.username(size=32) }}<br>
+            {{ form.username(size=32, autofocus=true) }}<br>
             {% for error in form.username.errors %}
             <span style="color: red;">[{{ error }}]</span>
             {% endfor %}
@@ -33,4 +33,4 @@
     {% endif %}
 </div>
 
-{% endblock %}
+{% endblock %}

app/templates/video.html

@@ -76,7 +76,7 @@
             </div>
             <div class="ui horizontal segments">
             <div class="center aligned ui segment">
-                <a href="{{ url_for('channel', id=info.uploader_id)}}">
+                <a href="{{ url_for('channel', id=info.channel_id)}}">
                     <i class="user icon"></i> <b>{{info.uploader}}</b>
                 </a>
                     <div class="label">

app/templates/ytsearch.html

@@ -4,7 +4,7 @@
 <div class="ui center aligned text container">
     <form action="{{url_for('ytsearch', _method='GET')}}">
     <div class="ui search">
-            <input class="prompt" name="q" type="text" placeholder="Search...">
+            <input class="prompt" name="q" type="text" placeholder="Search..." autofocus>
             <select name="s" id="sort">
                 <option value="0">Relevance</option>
                 <option value="3">Views</option>
@@ -19,13 +19,13 @@
     {% if results.channels %}
         <h3 class="ui dividing header">Users</h3>
     {% endif %}
-    <div class="ui relaxed divided list">        
+    <div class="ui relaxed divided list">
             {% for res in results.channels %}
                     <div class="item">
                         <div class="image">
                             {% if config.isInstance %}
                                 <img src="{{res.thumbnail}}" alt="Avatar">
-                            {% else %}    
+                            {% else %}
                                 <img alt="Avatar" src="{{ url_for('img', url=res.thumbnail) }}">
                             {% endif %}
                         </div>
@@ -45,7 +45,7 @@
                             <div class="ui label">
                                 <i class="video icon"></i> {{res.videos}}
                             </div>
-                            
+
                             {% if restricted or current_user.is_authenticated %}
                                 <div class="right floated content">
                                 {% if not current_user.is_following_yt(res.channelId) %}
@@ -59,7 +59,7 @@
                                         {{ btform.submit(value='Unfollow') }}
                                     </form>
                                 {% endif %}
-                                </div>  
+                                </div>
                             {% endif %}
                           </div>
                         </div>
@@ -94,4 +94,4 @@
     {%endif%}
 </div>
 </div>
-{% endblock %}
+{% endblock %}

docker-compose.yml

@@ -12,6 +12,18 @@ services:
       - mysql:/var/lib/mysql
     healthcheck:
       test: ["CMD", "mysqladmin", "ping", "--silent"]
+  nginx:
+    image: ytorg/nginx:latest
+    restart: unless-stopped
+    environment:
+      HOSTNAME: 'changeme.example.com'
+      HTTP_PORT: 8080
+      YOTTER_ADDRESS: 'http://yotter:5000'
+      YTPROXY_ADDRESS: 'http://unix:/var/run/ytproxy/http-proxy.sock'
+    ports:
+      - "127.0.0.1:8080:8080"
+    volumes:
+      - "/var/run/ytproxy:/app/socket/"
   ytproxy:
     image: 1337kavin/ytproxy:latest
     restart: unless-stopped
@@ -31,6 +43,10 @@ services:
     volumes:
       - migrations:/usr/src/app/migrations
       - ./yotter-config.json:/usr/src/app/yotter-config.json
+    healthcheck:
+      test: ["CMD", "wget" ,"--no-verbose", "--tries=1", "--spider", "http://localhost:5000"]
+      interval: 1m
+      timeout: 3s
 volumes:
   mysql:
   migrations:

nginx.Dockerfile

@@ -0,0 +1,12 @@
+FROM nginx:mainline-alpine
+
+WORKDIR /var/www
+COPY ./app/static ./static
+COPY ./nginx.conf.tmpl /nginx.conf.tmpl
+
+ENV HOSTNAME= \
+    HTTP_PORT=80 \
+    YOTTER_ADDRESS=http://127.0.0.1:5000 \
+    YTPROXY_ADDRESS=http://unix:/var/run/ytproxy/http-proxy.sock
+
+CMD ["/bin/sh", "-c", "envsubst '${HOSTNAME} ${HTTP_PORT} ${YOTTER_ADDRESS} ${YTPROXY_ADDRESS}' < /nginx.conf.tmpl > /etc/nginx/conf.d/default.conf && nginx -g 'daemon off;'"]

nginx.Dockerfile.dockerignore

@@ -0,0 +1,10 @@
+.circleci
+.git
+.github
+.gitignore
+cache
+Dockerfile
+docker-compose.yml
+LICENSE
+*.md
+dockerhash.txt

nginx.conf.tmpl

@@ -0,0 +1,30 @@
+server {
+  listen ${HTTP_PORT};
+  server_name ${HOSTNAME};
+  access_log off;
+
+  location / {
+    proxy_pass ${YOTTER_ADDRESS};
+    proxy_set_header Host $host;
+    proxy_http_version 1.1;
+    proxy_set_header Connection "";
+  }
+
+  location /static/ {
+    root /var/www;
+    sendfile on;
+    aio threads=default;
+  }
+
+  location ~ (^/videoplayback$|/videoplayback/|/vi/|/a/|/ytc|/vi_webp/|/sb/) {
+    proxy_pass ${YTPROXY_ADDRESS};
+    add_header Access-Control-Allow-Origin *;
+    sendfile on;
+    tcp_nopush on;
+    aio_write on;
+    aio threads=default;
+    directio 512;
+    proxy_http_version 1.1;
+    proxy_set_header Connection "";
+  }
+}

nitter/feed.py

@@ -22,13 +22,13 @@ def get_feed(usernames, daysMaxOld=10, includeRT=True):
     '''
     feedTweets = []
     with FuturesSession() as session:
-        futures = [session.get('{instance}{user}'.format(instance=config['nitterInstance'], user=u)) for u in usernames]
+        futures = [session.get(f'{config["nitterInstance"]}{u}') for u in usernames]
         for future in as_completed(futures):
             res = future.result().content.decode('utf-8')
             html = BeautifulSoup(res, "html.parser")
             feedPosts = user.get_feed_tweets(html)
             feedTweets.append(feedPosts)
-    
+
     userFeed = []
     for feed in feedTweets:
         if not includeRT:
@@ -45,5 +45,6 @@ def get_feed(usernames, daysMaxOld=10, includeRT=True):
                 userFeed.remove(uf)
         userFeed.sort(key=lambda item:item['timeStamp'], reverse=True)
     except:
+        print("Error sorting feed - nitter/feed.py")
         return userFeed
-    return userFeed
+    return userFeed

nitter/user.py

@@ -19,7 +19,7 @@ config = json.load(open('yotter-config.json'))
 config['nitterInstance']
 
 def get_user_info(username):
-    response = urllib.request.urlopen('{instance}{user}'.format(instance=config['nitterInstance'], user=username)).read()
+    response = urllib.request.urlopen(f'{config["nitterInstance"]}{username}').read()
     #rssFeed = feedparser.parse(response.content)
 
     html = BeautifulSoup(str(response), "lxml")
@@ -32,7 +32,7 @@ def get_user_info(username):
             fullName = html.find('a', attrs={'class':'profile-card-fullname'}).getText().encode('latin1').decode('unicode_escape').encode('latin1').decode('utf8')
         else:
             fullName = None
-        
+
         if html.find('div', attrs={'class':'profile-bio'}):
             profileBio = html.find('div', attrs={'class':'profile-bio'}).getText().encode('latin1').decode('unicode_escape').encode('latin1').decode('utf8')
         else:
@@ -46,12 +46,12 @@ def get_user_info(username):
             "following":html.find_all('span', attrs={'class':'profile-stat-num'})[1].string,
             "followers":numerize.numerize(int(html.find_all('span', attrs={'class':'profile-stat-num'})[2].string.replace(",",""))),
             "likes":html.find_all('span', attrs={'class':'profile-stat-num'})[3].string,
-            "profilePic":"{instance}{pic}".format(instance=config['nitterInstance'], pic=html.find('a', attrs={'class':'profile-card-avatar'})['href'][1:])
+            "profilePic":config['nitterInstance'] + html.find('a', attrs={'class':'profile-card-avatar'})['href'][1:],
         }
         return user
 
-def get_tweets(user, page=1):        
-    feed = urllib.request.urlopen('{instance}{user}'.format(instance=config['nitterInstance'], user=user)).read()
+def get_tweets(user, page=1):
+    feed = urllib.request.urlopen(f'{config["nitterInstance"]}{user}').read()
     #Gather feedPosts
     res = feed.decode('utf-8')
     html = BeautifulSoup(res, "html.parser")
@@ -59,8 +59,9 @@ def get_tweets(user, page=1):
 
     if page == 2:
         nextPage = html.find('div', attrs={'class':'show-more'}).find('a')['href']
-        print('{instance}{user}{page}'.format(instance=config['nitterInstance'], user=user, page=nextPage))
-        feed = urllib.request.urlopen('{instance}{user}{page}'.format(instance=config['nitterInstance'], user=user, page=nextPage)).read()
+        url = f'{config["nitterInstance"]}{user}{nextPage}'
+        print(url)
+        feed = urllib.request.urlopen(url).read()
         res = feed.decode('utf-8')
         html = BeautifulSoup(res, "html.parser")
         feedPosts = get_feed_tweets(html)
@@ -96,17 +97,17 @@ def get_feed_tweets(html):
             tweet['timeStamp'] = str(datetime.datetime.strptime(date_time_str, '%d/%m/%Y %H:%M:%S'))
             tweet['date'] = post.find('span', attrs={'class':'tweet-date'}).find('a').text
             tweet['content'] = Markup(yotterify(post.find('div',  attrs={'class':'tweet-content'}).decode_contents().replace("\n", "<br>")))
-            
+
             if post.find('div', attrs={'class':'retweet-header'}):
                 tweet['username'] = post.find('div', attrs={'class':'retweet-header'}).find('div', attrs={'class':'icon-container'}).text
                 tweet['isRT'] = True
             else:
                 tweet['username'] = tweet['op']
                 tweet['isRT'] = False
-            
+
             tweet['profilePic'] = config['nitterInstance']+post.find('a', attrs={'class':'tweet-avatar'}).find('img')['src'][1:]
             tweet['url'] = config['nitterInstance'] + post.find('a', attrs={'class':'tweet-link'})['href'][1:]
-            
+
             # Is quoting another tweet
             if post.find('div', attrs={'class':'quote'}):
                 tweet['isReply'] = True
@@ -123,7 +124,7 @@ def get_feed_tweets(html):
                             tweet['replyingTweetContent'] = Markup(quote.find('div',  attrs={'class':'quote-text'}).replace("\n", "<br>"))
                         except:
                             tweet['replyingTweetContent'] = Markup(quote.find('div',  attrs={'class':'quote-text'}))
-                        
+
                     if quote.find('a', attrs={'class':'still-image'}):
                         tweet['replyAttachedImages'] = []
                         images = quote.find_all('a',  attrs={'class':'still-image'})
@@ -135,7 +136,7 @@ def get_feed_tweets(html):
                     post.find('div', attrs={'class':'quote'}).decompose()
             else:
                 tweet['isReply'] = False
-            
+
             # Has attatchments
             if post.find('div',  attrs={'class':'attachments'}):
                 # Images
@@ -167,8 +168,8 @@ def get_feed_tweets(html):
                     elif 'heart' in str(stat):
                         tweet['likes'] = stat.find('div',attrs={'class':'icon-container'}).text
                     else:
-                        tweet['quotes'] =  stat.find('div',attrs={'class':'icon-container'}).text              
+                        tweet['quotes'] =  stat.find('div',attrs={'class':'icon-container'}).text
             feedPosts.append(tweet)
     else:
         return {"emptyFeed": True}
-    return feedPosts
+    return feedPosts

pypy.Dockerfile

@@ -9,9 +9,15 @@ COPY ./requirements.txt /usr/src/app
 
 # Build Dependencies
 RUN apt-get update \
-    && apt-get install -yq build-essential libssl-dev libffi-dev libxml2-dev libxslt-dev zlib1g-dev \
+    && apt-get install -yq build-essential libssl-dev libffi-dev libxml2-dev libxslt-dev zlib1g-dev curl \
     && rm -rf /var/lib/apt/lists/* /etc/apt/sources.list.d/*
 
+# install rust toolchain
+RUN curl https://sh.rustup.rs -sSf | \
+    sh -s -- --default-toolchain stable -y
+
+ENV PATH=/root/.cargo/bin:$PATH
+
 # Python Dependencies
 RUN pip install --no-warn-script-location --ignore-installed --no-cache-dir --prefix=/install wheel cryptography gunicorn pymysql
 RUN pip install --no-warn-script-location --ignore-installed --no-cache-dir --prefix=/install -r requirements.txt

requirements.txt

@@ -1,8 +1,8 @@
 alembic==1.4.3
 beautifulsoup4==4.9.3
-bleach==3.2.1
-cachetools==4.1.1
-certifi==2020.11.8
+bleach==3.3.0
+cachetools==4.2.0
+certifi==2020.12.5
 chardet==3.0.4
 click==7.1.2
 feedparser==6.0.2
@@ -16,29 +16,29 @@ gevent==20.9.0
 greenlet==0.4.17
 idna==2.10
 itsdangerous==1.1.0
-Jinja2==2.11.2
-lxml==4.6.1
+Jinja2==2.11.3
+lxml>=4.6.3
 Mako==1.1.3
 MarkupSafe==1.1.1
 numerize==0.12
-packaging==20.4
+packaging==20.8
 pyparsing==2.4.7
 PySocks==1.7.1
 python-dateutil==2.8.1
 python-dotenv==0.15.0
 python-editor==1.0.4
-requests==2.25.0
+requests==2.25.1
 requests-futures==1.0.0
 sgmllib3k==1.0.0
 six==1.15.0
 socks==0
 soupsieve==2.0.1
-SQLAlchemy==1.3.20
-urllib3==1.26.2
+SQLAlchemy==1.3.22
+urllib3==1.26.5
 webencodings==0.5.1
 Werkzeug==1.0.1
 WTForms==2.3.3
 youtube-dlc==2020.11.11.post3
 youtube-search-fork==1.2.5
 zope.event==4.5.0
-zope.interface==5.1.2
+zope.interface==5.2.0

yotter-config.json

@@ -1,7 +1,7 @@
 {
 	"serverName": "yotter.xyz",
-	"nitterInstance": "https://nitter.net/",
-	"maxInstanceUsers": 120,
+	"nitterInstance": "https://nitter.mastodont.cat/",
+	"maxInstanceUsers": 200,
 	"serverLocation": "Germany",
 	"restrictPublicUsage":true,
 	"isInstance":true,

youtube/channel.py

@@ -105,25 +105,36 @@ def channel_ctoken_v1(channel_id, page, sort, tab, view=1):
 
     return base64.urlsafe_b64encode(pointless_nest).decode('ascii')
 
-def get_channel_tab_info(channel_id, page="1", sort=3, tab='videos', view=1, print_status=True):
+def get_channel_tab(channel_id, page="1", sort=3, tab='videos', view=1,
+                    ctoken=None, print_status=True):
     message = 'Got channel tab' if print_status else None
 
-    if int(sort) == 2 and int(page) > 1:
-        ctoken = channel_ctoken_v1(channel_id, page, sort, tab, view)
-        ctoken = ctoken.replace('=', '%3D')
-        url = ('https://www.youtube.com/channel/' + channel_id + '/' + tab
-            + '?action_continuation=1&continuation=' + ctoken
-            + '&pbj=1')
-        content = util.fetch_url(url, headers_desktop + real_cookie,
-            debug_name='channel_tab', report_text=message)
-    else:
+    if not ctoken:
         ctoken = channel_ctoken_v3(channel_id, page, sort, tab, view)
         ctoken = ctoken.replace('=', '%3D')
-        url = 'https://www.youtube.com/browse_ajax?ctoken=' + ctoken
-        content = util.fetch_url(url,
-            headers_desktop + generic_cookie,
-            debug_name='channel_tab', report_text=message)
 
+    # Not sure what the purpose of the key is or whether it will change
+    # For now it seems to be constant for the API endpoint, not dependent
+    # on the browsing session or channel
+    key = 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8'
+    url = 'https://www.youtube.com/youtubei/v1/browse?key=' + key
+
+    data = {
+        'context': {
+            'client': {
+                'hl': 'en',
+                'gl': 'US',
+                'clientName': 'WEB',
+                'clientVersion': '2.20180830',
+            },
+        },
+        'continuation': ctoken,
+    }
+
+    content_type_header = (('Content-Type', 'application/json'),)
+    content = util.fetch_url(
+        url, headers_desktop + content_type_header,
+        data=json.dumps(data), debug_name='channel_tab', report_text=message)
     info = yt_data_extract.extract_channel_info(json.loads(content), tab)
     if info['error'] is not None:
         return False
@@ -174,12 +185,31 @@ def get_number_of_videos_general(base_url):
     return get_number_of_videos_channel(get_channel_id(base_url))
 
 def get_channel_search_json(channel_id, query, page):
-    params = proto.string(2, 'search') + proto.string(15, str(page))
+    offset = proto.unpadded_b64encode(proto.uint(3, (page-1)*30))
+    params = proto.string(2, 'search') + proto.string(15, offset)
     params = proto.percent_b64encode(params)
     ctoken = proto.string(2, channel_id) + proto.string(3, params) + proto.string(11, query)
     ctoken = base64.urlsafe_b64encode(proto.nested(80226972, ctoken)).decode('ascii')
 
-    polymer_json = util.fetch_url("https://www.youtube.com/browse_ajax?ctoken=" + ctoken, headers_desktop, debug_name='channel_search')
+    key = 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8'
+    url = 'https://www.youtube.com/youtubei/v1/browse?key=' + key
+
+    data = {
+        'context': {
+            'client': {
+                'hl': 'en',
+                'gl': 'US',
+                'clientName': 'WEB',
+                'clientVersion': '2.20180830',
+            },
+        },
+        'continuation': ctoken,
+    }
+
+    content_type_header = (('Content-Type', 'application/json'),)
+    polymer_json = util.fetch_url(
+        url, headers_desktop + content_type_header,
+        data=json.dumps(data), debug_name='channel_search')
 
     return polymer_json
 
@@ -258,5 +288,3 @@ def get_channel_page_general_url(base_url, tab, request, channel_id=None):
         parameters_dictionary = request.args,
         **info
     )
-
-

youtube/channels.py

@@ -155,13 +155,13 @@ def get_info_grid_video_item(item, channel=None):
         'timeStamp':published,
         'duration':duration,
         'channelName':channel['username'],
-        'authorUrl':"/channel/{}".format(channel['channelId']),
+        'authorUrl':f"/channel/{channel['channelId']}",
         'channelId':channel['channelId'],
         'id':item['videoId'],
-        'videoUrl':"/watch?v={}".format(item['videoId']),
+        'videoUrl':f"/watch?v={item['videoId']}",
         'isLive':isLive,
         'isUpcoming':isUpcoming,
-        'videoThumb':item['thumbnail']['thumbnails'][0]['url']
+        'videoThumb':item['thumbnail']['thumbnails'][0]['url'],
     }
     return video
 
@@ -172,18 +172,18 @@ def get_author_info_from_channel(content):
     channel = {
         "channelId": cmd['channelId'],
         "username": cmd['title'],
-        "thumbnail": "https:{}".format(cmd['avatar']['thumbnails'][0]['url'].replace("/", "~")),
+        "thumbnail": f"https:{cmd['avatar']['thumbnails'][0]['url'].replace('/', '~')}",
         "description":description,
         "suscribers": cmd['subscriberCountText']['runs'][0]['text'].split(" ")[0],
-        "banner": cmd['banner']['thumbnails'][0]['url']
+        "banner": cmd['banner']['thumbnails'][0]['url'],
     }
     return channel
 
 def get_channel_info(channelId, videos=True, page=1, sort=3):
     if id_or_username(channelId) == "channel":
         videos = []
-        ciUrl = "https://www.youtube.com/channel/{}".format(channelId)
-        mainUrl = "https://www.youtube.com/browse_ajax?ctoken={}".format(channel_ctoken_desktop(channelId, page, sort, "videos"))
+        ciUrl = f"https://www.youtube.com/channel/{channelId}"
+        mainUrl = f"https://www.youtube.com/browse_ajax?ctoken={channel_ctoken_desktop(channelId, page, sort, 'videos')}"
         content = json.loads(requests.get(mainUrl, headers=headers).text)
         req = requests.get(ciUrl, headers=headers).text
 
@@ -210,4 +210,4 @@ def get_channel_info(channelId, videos=True, page=1, sort=3):
             return {"channel":authorInfo}
 
     else:
-        baseUrl = "https://www.youtube.com/user/{}".format(channelId)
+        baseUrl = f"https://www.youtube.com/user/{channelId}"

youtube/comments.py

@@ -21,7 +21,7 @@ from youtube.util import concat_or_none
 def make_comment_ctoken(video_id, sort=0, offset=0, lc='', secret_key=''):
     video_id = proto.as_bytes(video_id)
     secret_key = proto.as_bytes(secret_key)
-    
+
 
     page_info = proto.string(4,video_id) + proto.uint(6, sort)
     offset_information = proto.nested(4, page_info) + proto.uint(5, offset)
@@ -35,11 +35,11 @@ def make_comment_ctoken(video_id, sort=0, offset=0, lc='', secret_key=''):
     result = proto.nested(2, page_params) + proto.uint(3,6) + proto.nested(6, offset_information)
     return base64.urlsafe_b64encode(result).decode('ascii')
 
-def comment_replies_ctoken(video_id, comment_id, max_results=500):  
+def comment_replies_ctoken(video_id, comment_id, max_results=500):
 
     params = proto.string(2, comment_id) + proto.uint(9, max_results)
     params = proto.nested(3, params)
-    
+
     result = proto.nested(2, proto.string(2, video_id)) + proto.uint(3,6) + proto.nested(6, params)
     return base64.urlsafe_b64encode(result).decode('ascii')
 

youtube/playlist.py

@@ -14,15 +14,15 @@ import flask
 
 
 
-def playlist_ctoken(playlist_id, offset):  
-    
+def playlist_ctoken(playlist_id, offset):
+
     offset = proto.uint(1, offset)
     # this is just obfuscation as far as I can tell. It doesn't even follow protobuf
     offset = b'PT:' + proto.unpadded_b64encode(offset)
     offset = proto.string(15, offset)
 
     continuation_info = proto.string( 3, proto.percent_b64encode(offset) )
-    
+
     playlist_id = proto.string(2, 'VL' + playlist_id )
     pointless_nest = proto.string(80226972, playlist_id + continuation_info)
 
@@ -51,7 +51,7 @@ def playlist_first_page(playlist_id, report_text = "Retrieved playlist"):
     content = json.loads(util.uppercase_escape(content.decode('utf-8')))
 
     return content
-    
+
 
 #https://m.youtube.com/playlist?itct=CBMQybcCIhMIptj9xJaJ2wIV2JKcCh3Idwu-&ctoken=4qmFsgI2EiRWTFBMT3kwajlBdmxWWlB0bzZJa2pLZnB1MFNjeC0tN1BHVEMaDmVnWlFWRHBEUWxFJTNE&pbj=1
 def get_videos(playlist_id, page):

youtube/proto.py

@@ -5,13 +5,13 @@ import io
 def byte(n):
     return bytes((n,))
 
-    
+
 def varint_encode(offset):
     '''In this encoding system, for each 8-bit byte, the first bit is 1 if there are more bytes, and 0 is this is the last one.
     The next 7 bits are data. These 7-bit sections represent the data in Little endian order. For example, suppose the data is
     aaaaaaabbbbbbbccccccc (each of these sections is 7 bits). It will be encoded as:
     1ccccccc 1bbbbbbb 0aaaaaaa
-    
+
     This encoding is used in youtube parameters to encode offsets and to encode the length for length-prefixed data.
     See https://developers.google.com/protocol-buffers/docs/encoding#varints for more info.'''
     needed_bytes = ceil(offset.bit_length()/7) or 1 # (0).bit_length() returns 0, but we need 1 in that case.
@@ -20,20 +20,20 @@ def varint_encode(offset):
         encoded_bytes[i] = (offset & 127) | 128  # 7 least significant bits
         offset = offset >> 7
     encoded_bytes[-1] = offset & 127 # leave first bit as zero for last byte
-    
+
     return bytes(encoded_bytes)
 
-    
+
 def varint_decode(encoded):
     decoded = 0
     for i, byte in enumerate(encoded):
         decoded |= (byte & 127) << 7*i
-        
+
         if not (byte & 128):
             break
     return decoded
 
-    
+
 def string(field_number, data):
     data = as_bytes(data)
     return _proto_field(2, field_number, varint_encode(len(data)) + data)
@@ -41,20 +41,20 @@ nested = string
 
 def uint(field_number, value):
     return _proto_field(0, field_number, varint_encode(value))
-    
 
-    
-    
+
+
+
 def _proto_field(wire_type, field_number, data):
     ''' See https://developers.google.com/protocol-buffers/docs/encoding#structure '''
     return varint_encode( (field_number << 3) | wire_type) + data
 
 
-    
+
 def percent_b64encode(data):
     return base64.urlsafe_b64encode(data).replace(b'=', b'%3D')
-    
-    
+
+
 def unpadded_b64encode(data):
     return base64.urlsafe_b64encode(data).replace(b'=', b'')
 
@@ -81,7 +81,7 @@ def read_varint(data):
         i += 1
     return result
 
-                                
+
 def read_group(data, end_sequence):
     start = data.tell()
     index = data.original.find(end_sequence, start)
@@ -101,7 +101,7 @@ def read_protobuf(data):
             break
         wire_type = tag & 7
         field_number = tag >> 3
-        
+
         if wire_type == 0:
             value = read_varint(data)
         elif wire_type == 1:

youtube/search.py

@@ -61,7 +61,7 @@ def get_channel_renderer_item_info(item):
         suscribers = item['subscriberCountText']['simpleText'].split(" ")[0]
     except:
         suscribers = "?"
-    
+
     try:
         description = utils.get_description_snippet_text(item['descriptionSnippet']['runs'])
     except KeyError:
@@ -159,10 +159,9 @@ def get_video_renderer_item_info(item):
         'authorUrl':"/channel/{}".format(item['ownerText']['runs'][0]['navigationEndpoint']['browseEndpoint']['browseId']),
         'channelId':item['ownerText']['runs'][0]['navigationEndpoint']['browseEndpoint']['browseId'],
         'id':item['videoId'],
-        'videoUrl':"/watch?v={}".format(item['videoId']),
+        'videoUrl':f"/watch?v={item['videoId']}",
         'isLive':isLive,
         'isUpcoming':isUpcoming,
-        'videoThumb':item['thumbnail']['thumbnails'][0]['url']
+        'videoThumb':item['thumbnail']['thumbnails'][0]['url'],
     }
     return video
-

youtube/util.py

@@ -120,9 +120,9 @@ def fetch_url_response(url, headers=(), timeout=15, data=None,
     if data is not None:
         method = "POST"
         if isinstance(data, str):
-            data = data.encode('ascii')
+            data = data.encode('utf-8')
         elif not isinstance(data, bytes):
-            data = urllib.parse.urlencode(data).encode('ascii')
+            data = urllib.parse.urlencode(data).encode('utf-8')
 
     if cookiejar_send is not None or cookiejar_receive is not None:     # Use urllib
         req = urllib.request.Request(url, data=data, headers=headers)
@@ -143,7 +143,7 @@ def fetch_url_response(url, headers=(), timeout=15, data=None,
         else:
             retries = urllib3.Retry(3)
         pool = get_pool(use_tor)
-        response = pool.request(method, url, headers=headers,
+        response = pool.request(method, url, headers=headers, body=data,
                                 timeout=timeout, preload_content=False,
                                 decode_content=False, retries=retries)
         cleanup_func = (lambda r: r.release_conn())
@@ -156,7 +156,7 @@ def fetch_url(url, headers=(), timeout=15, report_text=None, data=None,
     start_time = time.time()
 
     response, cleanup_func = fetch_url_response(
-        url, headers, timeout=timeout,
+        url, headers, timeout=timeout, data=data,
         cookiejar_send=cookiejar_send, cookiejar_receive=cookiejar_receive,
         use_tor=use_tor)
     response_time = time.time()
@@ -304,7 +304,7 @@ def video_id(url):
 # default, sddefault, mqdefault, hqdefault, hq720
 def get_thumbnail_url(video_id):
     return "/i.ytimg.com/vi/" + video_id + "/mqdefault.jpg"
-    
+
 def seconds_to_timestamp(seconds):
     seconds = int(seconds)
     hours, seconds = divmod(seconds,3600)
@@ -394,4 +394,3 @@ def check_gevent_exceptions(*tasks):
     for task in tasks:
         if task.exception:
             raise task.exception
-

youtube/utils.py

@@ -29,7 +29,7 @@ def parse_comment(raw_comment):
     cmnt = {}
     imgHostName = urllib.parse.urlparse(raw_comment['author_avatar'][1:]).netloc
     cmnt['author'] = raw_comment['author']
-    cmnt['thumbnail'] = raw_comment['author_avatar'].replace("https://{}".format(imgHostName),"")[1:] + "?host=" + imgHostName
+    cmnt['thumbnail'] = raw_comment['author_avatar'].replace(f"https://{imgHostName}","")[1:] + "?host=" + imgHostName
 
     print(cmnt['thumbnail'])
     cmnt['channel'] = raw_comment['author_url']
@@ -58,4 +58,4 @@ def post_process_comments_info(comments_info):
     comments = []
     for comment in comments_info['comments']:
         comments.append(parse_comment(comment))
-    return comments
+    return comments

youtube/video.py

@@ -32,7 +32,7 @@ def get_info(url):
         video['subtitles'] = info['subtitles']
         video['duration'] = info['duration']
         video['view_count'] = info['view_count']
-        
+
         if(info['like_count'] is None):
             video['like_count'] = 0
         else:
@@ -75,4 +75,4 @@ def get_video_formats(formats, audio=False):
     if audio:
         return audio_formats
     else:
-        return best_formats
+        return best_formats

youtube/watch.py

@@ -266,5 +266,3 @@ def format_bytes(bytes):
     suffix = ['B', 'KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB', 'ZiB', 'YiB'][exponent]
     converted = float(bytes) / float(1024 ** exponent)
     return '%.2f%s' % (converted, suffix)
-
-

youtube/yt_data_extract/common.py

@@ -290,7 +290,7 @@ def extract_item_info(item, additional_info={}):
         info['duration'] = extract_str(item.get('lengthText'))
 
         # if it's an item in a playlist, get its index
-        if 'index' in item: # url has wrong index on playlist page 
+        if 'index' in item: # url has wrong index on playlist page
             info['index'] = extract_int(item.get('index'))
         elif 'indexText' in item:
             # Current item in playlist has ▶ instead of the actual index, must
@@ -329,6 +329,11 @@ def extract_item_info(item, additional_info={}):
 
 def extract_response(polymer_json):
     '''return response, error'''
+    # /youtubei/v1/browse endpoint returns response directly
+    if isinstance(polymer_json, dict) and 'responseContext' in polymer_json:
+        # this is the response
+        return polymer_json, None
+
     response = multi_deep_get(polymer_json, [1, 'response'], ['response'])
     if response is None:
         return None, 'Failed to extract response'

youtube/yt_data_extract/watch_extraction.py

@@ -172,14 +172,13 @@ def _extract_watch_info_mobile(top_level):
         else:
             info['playlist'] = {}
             info['playlist']['title'] = playlist.get('title')
-            info['playlist']['author'] = extract_str(multi_get(playlist, 
+            info['playlist']['author'] = extract_str(multi_get(playlist,
                 'ownerName', 'longBylineText', 'shortBylineText', 'ownerText'))
             author_id = deep_get(playlist, 'longBylineText', 'runs', 0,
                 'navigationEndpoint', 'browseEndpoint', 'browseId')
             info['playlist']['author_id'] = author_id
-            if author_id:
-                info['playlist']['author_url'] = concat_or_none(
-                    'https://www.youtube.com/channel/', author_id)
+            info['playlist']['author_url'] = concat_or_none(
+                'https://www.youtube.com/channel/', author_id)
             info['playlist']['id'] = playlist.get('playlistId')
             info['playlist']['url'] = concat_or_none(
                 'https://www.youtube.com/playlist?list=',
@@ -447,7 +446,8 @@ def _extract_playability_error(info, player_response, error_prefix=''):
 
 SUBTITLE_FORMATS = ('srv1', 'srv2', 'srv3', 'ttml', 'vtt')
 def extract_watch_info(polymer_json):
-    info = {'playability_error': None, 'error': None}
+    info = {'playability_error': None, 'error': None,
+        'player_response_missing': None}
 
     if isinstance(polymer_json, dict):
         top_level = polymer_json
@@ -509,6 +509,10 @@ def extract_watch_info(polymer_json):
     if not info['formats']:
         _extract_formats(info, player_response)
 
+    # see https://github.com/user234683/youtube-local/issues/22#issuecomment-706395160
+    info['player_urls_missing'] = (
+        not info['formats'] and not embedded_player_response)
+
     # playability errors
     _extract_playability_error(info, player_response)
 
@@ -565,6 +569,84 @@ def extract_watch_info(polymer_json):
     info['author_url'] = 'https://www.youtube.com/channel/' + info['author_id'] if info['author_id'] else None
     return info
 
+single_char_codes = {
+    'n': '\n',
+    '\\': '\\',
+    '"': '"',
+    "'": "'",
+    'b': '\b',
+    'f': '\f',
+    'n': '\n',
+    'r': '\r',
+    't': '\t',
+    'v': '\x0b',
+    '0': '\x00',
+    '\n': '', # backslash followed by literal newline joins lines
+}
+def js_escape_replace(match):
+    r'''Resolves javascript string escape sequences such as \x..'''
+    # some js-strings in the watch page html include them for no reason
+    # https://mathiasbynens.be/notes/javascript-escapes
+    escaped_sequence = match.group(1)
+    if escaped_sequence[0] in ('x', 'u'):
+        return chr(int(escaped_sequence[1:], base=16))
+
+    # In javascript, if it's not one of those escape codes, it's just the
+    # literal character. e.g., "\a" = "a"
+    return single_char_codes.get(escaped_sequence, escaped_sequence)
+
+# works but complicated and unsafe:
+#PLAYER_RESPONSE_RE = re.compile(r'<script[^>]*?>[^<]*?var ytInitialPlayerResponse = ({(?:"(?:[^"\\]|\\.)*?"|[^"])+?});')
+
+# Because there are sometimes additional statements after the json object
+# so we just capture all of those until end of script and tell json decoder
+# to ignore extra stuff after the json object
+PLAYER_RESPONSE_RE = re.compile(r'<script[^>]*?>[^<]*?var ytInitialPlayerResponse = ({.*?)</script>')
+INITIAL_DATA_RE = re.compile(r"<script[^>]*?>var ytInitialData = '(.+?[^\\])';")
+BASE_JS_RE = re.compile(r'jsUrl":\s*"([\w\-\./]+?/base.js)"')
+JS_STRING_ESCAPE_RE = re.compile(r'\\([^xu]|x..|u....)')
+def extract_watch_info_from_html(watch_html):
+    base_js_match = BASE_JS_RE.search(watch_html)
+    player_response_match = PLAYER_RESPONSE_RE.search(watch_html)
+    initial_data_match = INITIAL_DATA_RE.search(watch_html)
+
+    if base_js_match is not None:
+        base_js_url = base_js_match.group(1)
+    else:
+        base_js_url = None
+
+    if player_response_match is not None:
+        decoder = json.JSONDecoder()
+        # this will make it ignore extra stuff after end of object
+        player_response = decoder.raw_decode(player_response_match.group(1))[0]
+    else:
+        return {'error': 'Could not find ytInitialPlayerResponse'}
+        player_response = None
+
+    if initial_data_match is not None:
+        initial_data = initial_data_match.group(1)
+        initial_data = JS_STRING_ESCAPE_RE.sub(js_escape_replace, initial_data)
+        initial_data = json.loads(initial_data)
+    else:
+        print('extract_watch_info_from_html: failed to find initialData')
+        initial_data = None
+
+    # imitate old format expected by extract_watch_info
+    fake_polymer_json = {
+        'player': {
+            'args': {},
+            'assets': {
+                'js': base_js_url
+            }
+        },
+        'playerResponse': player_response,
+        'response': initial_data,
+    }
+
+    return extract_watch_info(fake_polymer_json)
+
+
+
 def get_caption_url(info, language, format, automatic=False, translation_language=None):
     '''Gets the url for captions with the given language and format. If automatic is True, get the automatic captions for that language. If translation_language is given, translate the captions from `language` to `translation_language`. If automatic is true and translation_language is given, the automatic captions will be translated.'''
     url = info['_captions_base_url']
@@ -580,7 +662,8 @@ def get_caption_url(info, language, format, automatic=False, translation_languag
     return url
 
 def update_with_age_restricted_info(info, video_info_page):
-    ERROR_PREFIX = 'Error bypassing age-restriction: '
+    '''Inserts urls from 'player_response' in get_video_info page'''
+    ERROR_PREFIX = 'Error getting missing player or bypassing age-restriction: '
 
     video_info = urllib.parse.parse_qs(video_info_page)
     player_response = deep_get(video_info, 'player_response', 0)
@@ -603,7 +686,9 @@ def requires_decryption(info):
 # adapted from youtube-dl and invidious:
 # https://github.com/omarroth/invidious/blob/master/src/invidious/helpers/signatures.cr
 decrypt_function_re = re.compile(r'function\(a\)\{(a=a\.split\(""\)[^\}{]+)return a\.join\(""\)\}')
-op_with_arg_re = re.compile(r'[^\.]+\.([^\(]+)\(a,(\d+)\)')
+# gives us e.g. rt, .xK, 5 from rt.xK(a,5) or rt, ["xK"], 5 from rt["xK"](a,5)
+# (var, operation, argument)
+var_op_arg_re = re.compile(r'(\w+)(\.\w+|\["[^"]+"\])\(a,(\d+)\)')
 def extract_decryption_function(info, base_js):
     '''Insert decryption function into info. Return error string if not successful.
     Decryption function is a list of list[2] of numbers.
@@ -617,10 +702,11 @@ def extract_decryption_function(info, base_js):
     if not function_body:
         return 'Empty decryption function body'
 
-    var_name = get(function_body[0].split('.'), 0)
-    if var_name is None:
+    var_with_operation_match = var_op_arg_re.fullmatch(function_body[0])
+    if var_with_operation_match is None:
         return 'Could not find var_name'
 
+    var_name = var_with_operation_match.group(1)
     var_body_match = re.search(r'var ' + re.escape(var_name) + r'=\{(.*?)\};', base_js, flags=re.DOTALL)
     if var_body_match is None:
         return 'Could not find var_body'
@@ -649,13 +735,13 @@ def extract_decryption_function(info, base_js):
 
     decryption_function = []
     for op_with_arg in function_body:
-        match = op_with_arg_re.fullmatch(op_with_arg)
+        match = var_op_arg_re.fullmatch(op_with_arg)
         if match is None:
             return 'Could not parse operation with arg'
-        op_name = match.group(1)
+        op_name = match.group(2).strip('[].')
         if op_name not in operation_definitions:
-            return 'Unknown op_name: ' + op_name
-        op_argument = match.group(2)
+            return 'Unknown op_name: ' + str(op_name)
+        op_argument = match.group(3)
         decryption_function.append([operation_definitions[op_name], int(op_argument)])
 
     info['decryption_function'] = decryption_function