Bump @xmldom/xmldom from 0.7.5 to 0.7.8 in /webapp

.dockerignore

@@ -0,0 +1 @@
+videos/*

.github/workflows/release.yml

@@ -0,0 +1,17 @@
+# .github/workflows/release.yml
+name: Release
+
+on:
+  pull_request:
+    types: closed
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Tag
+        uses: K-Phoen/semver-release-action@master
+        with:
+          release_branch: master
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/test.yml

@@ -0,0 +1,65 @@
+name: Build and test
+
+on: [push, pull_request]
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Build the stack alternative origin
+        run: echo "nginx" | ./up.sh
+        if: always()
+      - name: Run Selenium tests
+        run: cd selenium && sleep 90 && sudo docker run --net horahora_default $(docker build -q .)
+        if: always()
+      - name: Build the stack localhost
+        run: echo "localhost" | ./up.sh
+        if: always()
+      - name: Run integration API tests
+        run: cd tests && go run main.go
+        if: always()
+      - name: print relevant logs (videoservice)
+        run: docker logs $(docker ps | grep "videoservice" | awk '{print $1}')
+        if: always()
+      - name: print relevant logs (userservice)
+        run: docker logs $(docker ps | grep "userservice" | awk '{print $1}')
+        if: always()
+      - name: print relevant logs (scheduler)
+        run: docker logs $(docker ps | grep "scheduler" | awk '{print $1}')
+        if: always()
+      - name: print relevant logs (postgres)
+        run: docker logs $(docker ps | grep "postgres" | awk '{print $1}')
+        if: always()
+      - name: print relevant logs (frontapi)
+        run: docker logs $(docker ps | grep "frontapi" | awk '{print $1}')
+        if: always()
+      - name: print relevant logs (nginx)
+        run: docker logs $(docker ps | grep "nginx" | awk '{print $1}')
+        if: always()
+  release:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Log in to the Container registry
+        uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+      - name: Build the stack localhost
+        run: echo "localhost" | ./up.sh
+        if: always()
+      - name: publish main
+        uses: pennsignals/[email protected]
+        with:
+          version: main
+          docker_compose: 'docker-compose.yml' # required

.gitignore

@@ -1 +1,25 @@
-youtube-dl*
+# CLI build output
+/horahora
+# CLI windows build output
+/horahora.exe
+
+# vscode files
+*.code-workspace
+
+# temporary folders created by golang watcher
+**/tmp/*
+
+youtube-dl*
+packer/vpn_conf*
+data*
+secrets.env
+sql/backup*
+/videos
+
+# env file is used for the docker compose
+/.env
+/.env*
+
+docker-compose.yaml
+webapp/.env
+configs/*

.vscode/extensions.json

@@ -0,0 +1,6 @@
+{
+  "recommendations": [
+    // https://go.dev/doc/editors#options
+    "golang.go",
+  ]
+}

.vscode/horahora.code-workspace.example

@@ -0,0 +1,95 @@
+{
+	"folders": [
+		{
+			"name": "horahora",
+			"path": "."
+		},
+		{
+			"name": "front_api",
+			"path": "front_api"
+		},
+		{
+			"name": "backup_service",
+			"path": "backup_service"
+		},
+		{
+			"name": "scheduler",
+			"path": "scheduler"
+		},
+		{
+			"name": "stomp_proxy",
+			"path": "stomp_proxy"
+		},
+		{
+			"name": "tests",
+			"path": "tests"
+		},
+		{
+			"name": "user_service",
+			"path": "user_service"
+		},
+		{
+			"name": "video_service",
+			"path": "video_service"
+		},
+		{
+			"name": "webapp",
+			"path": "webapp"
+		},
+		{
+      "name": "cli",
+      "path": "cli"
+    },
+		{
+			"name": "schema",
+			"path": "schema"
+		}
+	],
+	"settings": {
+		"files.exclude": {
+			"**/.git": true,
+			"**/.svn": true,
+			"**/.hg": true,
+			"**/CVS": true,
+			"**/.DS_Store": true,
+			"**/Thumbs.db": true,
+			"**/node_modules": true,
+			"front_api": true,
+			"backup_service": true,
+			"scheduler": true,
+			"stomp_proxy": true,
+			"tests": true,
+			"user_service": true,
+			"video_service": true,
+			"webapp": true,
+			"schema": true,
+			"cli": true
+		},
+		"files.associations": {
+			// syntax highlight for example env files
+			"**/.env*": "dotenv",
+			// syntax highlight for example workspace files
+			"**/*.code-workspace*": "jsonc"
+		},
+		// remove text 3 options if you've set them up in global editor settings
+		"files.trimTrailingWhitespace": true,
+  	"files.insertFinalNewline": true,
+  	"files.trimFinalNewlines": true
+	},
+	"extensions": {
+		"recommendations": [
+			// `.env` syntax highlight
+			"mikestead.dotenv",
+			// docker-related features plus highlight in dockerfiles and compose configs
+			"ms-azuretools.vscode-docker",
+			// lots of QoL git features
+			"eamodio.gitlens",
+			// nginx config highlight
+			"ahmadalli.vscode-nginx-conf",
+			// `.sql` formatter
+			"bradymholt.pgformatter",
+			// `.proto3` syntax highlight
+			"zxh404.vscode-proto3"
+		]
+	}
+}

.vscode/settings.json

@@ -0,0 +1,20 @@
+{
+  "gopls": {
+    "ui.semanticTokens": true
+  },
+  "[json]": {
+    "editor.tabSize": 2
+  },
+  "[jsonc]": {
+    "editor.tabSize": 2
+  },
+  "json.schemas": [
+    {
+      "fileMatch": ["schema/**/*schema.json"],
+      "url": "./schema/meta.schema.json"
+    }
+  ],
+  "files.trimTrailingWhitespace": true,
+  "files.insertFinalNewline": true,
+  "files.trimFinalNewlines": true
+}

CONTRIBUTING.md

@@ -0,0 +1,23 @@
+# Contributing
+Thank you for your interest in contributing to Horahora! Think you can do better than what we've done? You probably can!
+
+## Architectural Overview
+![](https://raw.githubusercontent.com/horahoradev/horahora/master/Architectural_Drawing.png)
+
+Horahora's architecture is microservice-based. The main microservices are:
+- `front_api`: which is the RESTful API to the rest of the services
+- `userservice`: which does all authentication and handles user storage/permissions
+- `videoservice`: which does all video storage, uploads to the origin (e.g. s3/backblaze), queries, transcoding, etc
+- `scheduler`: which handles content archival requests and downloads
+Communication between userservice, videosercvice, and scheduler is GRPC-based. For more details on individual microservice architecture, see the README for whichever microservice.
+
+Postgresql is used as the database for each service, but Redis is also used for very specific purposes (e.g. distributed locking). Schema migrations can be found within the "migrations" directory within each service. As an example, [here's the migrations directory for Videoservice](https://github.com/horahoradev/horahora/tree/master/video_service/migrations). Migrations are applied using Flyway as a means of providing schema versioning.
+
+Since microservices communicate via GRPC, the API for each service is defined by the domain specific proto3 language. [Here's an example.](https://github.com/horahoradev/horahora/blob/master/video_service/protocol/videoservice.proto) This file in particular defines the API for videoservice, which other services will use to invoke videoservice's functionality. We use this file to generate interface and struct definitions in Golang (our target language), and then implement every method. GRPC implementations for videoservice can be found here: https://github.com/horahoradev/horahora/blob/master/video_service/internal/grpcserver/grpc.go . [Here's a minimal method implementation example](https://github.com/horahoradev/horahora/blob/master/video_service/internal/grpcserver/grpc.go#L442), which rates videos: the implementation simply calls the AddRatingToVideoID method from the videos model (which is really more like an example of the repository pattern) with the supplied GRPC arguments, and returns a response.
+
+## Communication and Coordination
+Coordination happens via Discord, here: https://discord.gg/vfwfpctJRZ
+
+Just ping me (Otoman) with a message to the effect of what you're interested in doing, and I'll help you out as best I can.
+
+I've currently listed a few issues for the repo as "help-wanted" and "good first issue"; these represent good places to start, but if you don't see anything you like, or there's nothing that's unassigned, ping me, and I'll think of something that suits your goals.

Dockerfile

@@ -0,0 +1,18 @@
+FROM golang:1.17
+
+RUN apt-get update && \
+    apt install -y protobuf-compiler && \
+    go install google.golang.org/protobuf/cmd/[email protected] && \
+    go install google.golang.org/grpc/cmd/[email protected]
+
+RUN go env -w GO111MODULE="off" && \
+    go get github.com/envoyproxy/protoc-gen-validate && \
+    go env -w GO111MODULE=""
+
+RUN go install github.com/envoyproxy/[email protected]
+
+COPY gen_all.sh /bin/gen_all.sh
+
+WORKDIR /
+
+ENTRYPOINT ["/bin/gen_all.sh"]

Dockerfile.postgres

@@ -0,0 +1 @@
+postgres/Dockerfile

README.md

@@ -1,53 +1,77 @@
 # Horahora
-Horahora is a microservice-based video hosting website with additional functionality for content archival from Niconico, Bilibili, and Youtube. Users can upload their own content, or schedule categories of content from other websites to be archived (e.g. a given channel on Niconico, a tag on Youtube, or a playlist from Bilibili). Content archived from other websites will be accessible in the same manner as user-uploaded videos, and will be organized under the same metadata (author, tags) associated with the original video.
+## Locally archive, browse, and share videos from nearly any site
 
-This project is a WIP, and under active development. The MVP for local use is not yet complete, but nearly done. Contributions and suggestions are always welcome! If you have any questions regarding this project, feel free to contact me either via discord or email.
+Horahora is a collaborative archival management tool.
 
-Join our discord here: https://discord.gg/6TAEmAA
+It allows you to:
+- download and continuously sync videos from any link supported by yt-dlp
+- browse through downloaded videos by channel, tag, views, rating, upload date, etc
+- manage archival with a group of friends or untrusted users, with downloads being prioritized by the number of users subscribed to the video's category
+- manage site user permissions, ban users, delete videos, and view audit logs for admin/moderator actions
 
-## Architecture
-![](Architectural_Drawing.png)
+Join our Discord: https://discord.gg/vfwfpctJRZ
 
-Currently, there are three microservices:
-1. User Service, which handles registration, logins, and JWT validation
-2. Video Service, which handles video uploads (both from Scheduler and from users), transcoding/chunking as required for DASH, uploads to the origin, and storage of metadata.
-3. Scheduler, which handles content archival requests from users. For example, if a user specifies that they'd like all videos on Niconico with the tag "YTPMV" to be downloaded, Scheduler will download those videos, register them (and their associated creator) with video service and user service, and check that category of content regularly for new videos.
+![](https://github.com/horahoradev/horahora-designs/blob/master/Screenshot%20from%202022-10-09%2011-56-34.png?raw=true)
 
-For more in-depth information on a given microservice, consult its README.
+![](https://github.com/horahoradev/horahora-designs/blob/master/Screenshot%20from%202022-10-09%2011-54-48.png?raw=true)
 
-All microservices are horizontally scalable, containerized, and communicate via gRPC.
+![](https://github.com/horahoradev/horahora-designs/blob/master/Screenshot%20from%202022-10-09%2011-57-35.png?raw=true)
 
-The MVP will also consist of a frontend service to handle HTML templating, and a Censorship Service (name is a WIP) to manage the workflow for video approvals, and censorship of obscene content from foreign websites.  
+![](https://github.com/horahoradev/horahora-designs/blob/master/Screenshot%20from%202022-10-09%2011-57-52.png?raw=true)
 
-## How to Use
-Currently, only local use is supported.
-To run Horahora locally, follow these steps:
-1. First install the following dependencies:
-  - Flyway
-  - Docker
-  - Kubernetes
+Archival capabilities are provided by yt-dlp (a fork of youtube-dl).
 
-2. Start minikube with `minikube start --memory=3072`
-3. Run `./build.all.sh` (in the Kubernetes directory) to build all Docker images and send to the Docker daemon within minikube
-4. `./run-local.sh` to create Kubernetes deployments/services and apply database migrations. Keep running run-local.sh until migrations succeed.
-5. `./run-tests.sh` will run local integration tests.
-6. Navigate to `localhost:8080` to view the current state of the website.
+A word of warning: this application is pretty heavy, and setup can be complicated. If you're looking for something simpler, check out: https://github.com/tubearchivist/tubearchivist
 
-## Designs
-Designs are listed here:
-https://github.com/horahoradev/horahora-designs
-
-![](https://github.com/horahoradev/horahora-designs/blob/master/Login.png?raw=true)
-
-![](https://github.com/horahoradev/horahora-designs/blob/master/Video.png?raw=true)
+## Usage Instructions (START HERE)
 
-![](https://github.com/horahoradev/horahora-designs/blob/master/Profile.png?raw=true)
+1. Install docker, python3, and docker-compose
+2. ./up.sh
+   1. respond to requests for input
+2. Wait a minute, then visit localhost:80
+3. Login as admin/admin
+    - note that with the current video approval workflow, non-admin users won't be able to view unapproved videos
+    - it's recommended to visit /password-reset immediately to change the admin user's default password if using in an untrusted environment
+4. navigate to the archival requests page from the hamburger menu, add a link, and wait a few minutes
 
-## Task Roadmap
-Missing features are tracked using Trello.
-
-Our Trello board is:
-https://trello.com/b/Rm5TPR4Q/horahora
+That's it for basic usage, and should work. If that doesn't work, bug me on Discord.
 
+## Contributing
+Contributions are always welcome. Please see [CONTRIBUTING.md](https://github.com/horahoradev/horahora/blob/master/CONTRIBUTING.md) for details, including an architectural rundown.
 
+## Designs
+Designs are listed here:
+https://github.com/horahoradev/horahora-designs
 
+## More Detailed Feature List
+- performant at 170k videos, even for all varieties of search queries
+- support for videos which have been deleted from the origin (e.g. if the original site deletes the video, there's no impact on your instance)
+- support for comments, view count, user ratings
+- video approval workflow which prevents normal users from seeing videos before they've been approved
+- support for TOS/privacy policy
+- content archival modeled as one-to-many user subscriptions, so users "subscribe" to a category (link), and links are prioritized according to the number of subscribers
+- support for any website supported by yt-dlp which has the required metadata (but I only use YT/nicovideo atm)
+- artificial user creation: archived videos will be grouped under a Horahora user created for the archived website's user (e.g. if I archive from Russia Today, then a Russia TOday user will be created on Horahora)
+- support for yt-dlp tunneling via Gluetun (see below for setup)
+- dark mode toggle
+
+## Advanced Use Cases
+### Other Storage Backends (s3, backblaze, anything s3-compatible)
+By default, Horahora will store videos locally using Minio.
+
+If you don't want videos to be stored locally, modify .env, adding the relevant values for your use case.
+
+    - ORIGIN_FQDN: this will be the public URL of your Backblaze bucket WITH NO TRAILING SLASH. E.g. for me it's: https://f002.backblazeb2.com/file/otomads for backblaze, or https://horahora-dev-otomads.s3-us-west-1.amazonaws.com for s3.
+    - STORAGE_BACKEND: 'b2' or 's3' (depending on which you want to use)
+    - STORAGE_API_ID: the API ID for your Backblaze account if using backblaze, otherwise blank
+    - STORAGE_API_KEY: The API key for your Backblaze account, otherwise blank
+    - BUCKET_NAME: the storage bucket name for b2 or s3
+  If you want to use S3, you need to include your aws credentials and config in $HOME/.aws. The config and credentials will be mounted into the relevant services at runtime. See https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-files.html for more information.
+
+### Tunneling yt-dlp Traffic
+Horahora comes with Gluetun support out of the box. To enable it, you'll need to set the proper values in the "vpn config" section of the secrets.env.template file. This will enable your yt-dlp traffic to be tunneled through your VPN provider via a local Gluetun HTTP proxy.
+
+### Backup Restoration
+(this currently isn't functioning, I'll fix it later)
+
+Backup_service writes psql dumps of the three databases (userservice, videoservice, scheduler) to backblaze. To restore, place the three latest dumps in the sql dir, `docker-compose up`, run migrations, then run restore.sh from within the container.