Commit a278e2b0 authored by Melroy van den Berg's avatar Melroy van den Berg
Browse files

Add files

parents
Pipeline #1394 canceled with stage
module.exports = {
env: {
es6: true,
node: true,
mocha: true
},
extends: [
'standard'
],
globals: {
Atomics: 'readonly',
SharedArrayBuffer: 'readonly',
TelegramSecretHash: 'writable'
},
parserOptions: {
ecmaVersion: 2018,
sourceType: 'module'
},
rules: {
}
}
# Secret tokens
tokens.env
# Logs
logs
*.log
npm-debug.log*
yarn-debug.log*
yarn-error.log*
lerna-debug.log*
# Diagnostic reports (https://nodejs.org/api/report.html)
report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json
# Runtime data
pids
*.pid
*.seed
*.pid.lock
# Directory for instrumented libs generated by jscoverage/JSCover
lib-cov
# Coverage directory used by tools like istanbul
coverage
*.lcov
# nyc test coverage
.nyc_output
# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files)
.grunt
# Bower dependency directory (https://bower.io/)
bower_components
# node-waf configuration
.lock-wscript
# Compiled binary addons (https://nodejs.org/api/addons.html)
build/Release
# Dependency directories
node_modules/
jspm_packages/
# TypeScript v1 declaration files
typings/
# TypeScript cache
*.tsbuildinfo
# Optional npm cache directory
.npm
# Optional eslint cache
.eslintcache
# Optional REPL history
.node_repl_history
# Output of 'npm pack'
*.tgz
# Yarn Integrity file
.yarn-integrity
# dotenv environment variables file
.env
.env.test
# parcel-bundler cache (https://parceljs.org/)
.cache
# next.js build output
.next
# nuxt.js build output
.nuxt
# vuepress build output
.vuepress/dist
# Serverless directories
.serverless/
# FuseBox cache
.fusebox/
# DynamoDB Local files
.dynamodb/
before_script:
- npm install
cache:
key: "$CI_PIPELINE_ID"
untracked: true
paths:
- node_modules
lint:
stage: test
tags:
- test
script:
- npm run lint
This diff is collapsed.
# LBRY Channel RSS Feed
A [LBRY](https://lbry.com/) channel [RSS](https://en.wikipedia.org/wiki/RSS) feed provider.
Meaning, you can use your RSS reader to keep up-to-date with your favorite channels on LBRY.
## Creators
- Melroy van den Berg
## Develop
Requirements:
* [Node.js v10](https://nodejs.org/en/download/)
* npm (package manager)
* [Lbrynet deamon](https://github.com/lbryio/lbry-sdk/releases)
[//]: # ([Lbrycrd deamon](https://github.com/lbryio/lbrycrd) with txindex turned on)
```sh
curl -sL https://deb.nodesource.com/setup_10.x | sudo -E bash -
sudo apt-get install -y nodejs npm
```
### Running
```sh
RPC_PASSWORD=xyz
```
Start the web-server: `npm start` (or `node src/index.js`)
**Note 1:** Reverse proxy (eg. Nginx) is advised when you serve the server on the world-wide-web. Expose the webserver on port 443 (with SSL). See [nginx_example.conf](nginx_example.conf).
**Note 2:** Assuming you are running the lbrynet deamon (see requirements).
**Note 3:** Assuming you are running the lbrycrd deamon (see requirements), with JSON RPC enabled and txindex enabled. See LBRYcrd setup below.
### Linting
Run lint: `npm run lint`
Fix lint issues: `npm run fix`
### Unit Testing
Run test: `npm test`
## Production
Starting the bot, can be done via:
- `./start_rss_feed_prod.sh`
The bot can be started via crontab for example:
```sh
@reboot sh /path/to/start_rss_feed_prod.sh
```
**General setup:**
- Be-sure both `lbrycrdd` and `lbrynet` binaries are installed into `/usr/bin` directory!
- Create an user lbry the unix machine (`adduser -M lbry`)
### LBRYcrd setup
- Place the LBRYcrd file (`lbryd.conf`) in `/etc/lbry` for the LBRY Core Daemon service, example of this file:
```sh
rpcuser=lbry
rpcpassword=my_secure_password
daemon=1
server=1
txindex=1
```
- See [lbrycrd.service systemd file](lbrycrd.service) for Debian based distributions. Place this file into `/etc/systemd/system` folder.
- Core data will be stored into `/var/lib/lbrycrd`
### LBRYnet setup
- Place the LBRYNet file (`lbrynet.yml`) in `/etc/lbry`, example of this file:
```yml
api: 127.0.0.1:5279
streaming_server: 127.0.0.1:5280
allowed_origin: localhost
data_dir: /var/lib/lbrynet
download_dir: /var/lib/lbrynet
wallet_dir: /var/lib/lbryum
save_files: false
save_blobs: false
max_key_fee:
currency: LBC
amount: 0
use_keyring: false
```
- See [lbrynet.service systemd file](lbrynet.service). Place also inside `/etc/systemd/system`.
- LBRYNet (SDK) data will be stored into `/var/lib/lbrynet`
This diff is collapsed.
# Documentation
General documentation about LBRY (pronounced as 'Library'). LBRY is a decentralized blockchain-based content sharing application using the LBRY protocol specifiction.
Read the [lbry overview page](https://lbry.tech/overview).
I'll discuss the most important projects down below.
```sh
++
:+++++
+++++++++
'++++++++++++`
.++++++',++++++++`
+++++++ .++++++++.
;++++++: `++++++++,
+++++++ ++++++++,
+++++++` ++++++++:
,++++++' ++++++++;
+++++++ ++++++++'
'++++++, ++++++++'
`+++++++ '++++++++
+++++++ '++++++++
:++++++; ;++++++++
+++++++ :++++++++
'++++++. ,++++++++
.++++++' ,++++++++`
+++++++ .++++++++`
;++++++, `+++++++
`+++++++ `+++++
+++++++` ++++
:++++++; +++++
+++++++ ,++++++
'++++++. +++++++
.++++++' '++++++,
+++++++ `+++++++
+++++: +++++++
++++ ;++++++:
++++ +++++++
++++ ++ +++++++`
++++ ++++ ,++++++' .:
++++ ++++++ +++++++ :'++++++++++
++++ ++++++++ '++++++. ++++++++++
++++ :++++++++ .++++++' .+++++++++
++++ :++++++++ +++++++ ++++++++.
++++ ,++++++++ ;++++++: :++++++++
++++ ,++++++++ `+++++++ +++++++++,
++++ ,++++++++ +++++++` +++++++++++
+++++. .++++++++` :++++++; ,+++++++ +++;
+++++++. .++++++++` +++++++ +++++++ ++
++++++++, `++++++++` '++++++. '++++++: ,'
++++++++, `++++++++` .++++++' .+++++++
++++++++, `++++++++. +++++++ +++++++`
++++++++, ++++++++.'++++++, ;++++++;
++++++++: +++++++++++++ `+++++++
++++++++: +++++++++` +++++++.
++++++++: +++++; ,++++++'
++++++++: ++ +++++++
++++++++; '++++++,
'+++++++; .+++++++
'+++++++; +++++++
'+++++++; ;++++++:
'+++++++' `+++++++
'+++++++' +++++++`
'+++++++' :++++++;
;+++++++' +++++++
;+++++++++++++.
;+++++++++'
;++++++
:++,
```
## lbry-sdk
Most applications typically use Lbry-sdk, it implements the LBRY protocol specification (except the blockchain protocol), extra components for developers, deamon that participates in the LBRY data network and most importantly the convenient API for building apps on top of LBRY.
* [lbry-sdk API](https://lbry.tech/api/sdk) (using the lbry-sdk deamon API)
* [Download librynet](https://github.com/lbryio/lbry-sdk/releases)
Run the deamon, via: `./lbrynet start`
API CLI status request (example): `curl -d'{"method": "status", "params": {}}' http://localhost:5279/`
*Note:* lbrycrd is not required for the SDK, since it's using SPV (a scheme to validate transactions without storing the whole blockchain = lbryum) by default instead of the full blockchain/full node (lbrycrd).
## Chainquery
Chainquery takes all the raw data in the blockchain, decompresses and extracts it, and stores it in a relational database, all in real-time. This allows the rich features of SQL to be applied to live blockchain data.
Chainquery is programmed in the Go language, and requires lbrycrd.
* [Chainquery Github source-code](https://github.com/lbryio/chainquery)
* [Chainquery schema](https://github.com/lbryio/chainquery/blob/master/db/chainquery_schema.sql)
There is also a public API available, just two examples:
* Get the latest block information: `https://chainquery.lbry.com/api/sql?query=SELECT * FROM block ORDER BY height DESC LIMIT 1`
* Get all claims and their details for the name "lbry": `https://chainquery.lbry.com/api/sql?query=SELECT * FROM claim WHERE name = "lbry"`
## lbrycrd
The **official** implementation of the LBRY blockchain protocol. The core of LBRY.
* [lbrycrd Github source-code](https://github.com/lbryio/lbrycrd)
* [lbrycrd spec doc](lbry-spec.pdf)
* [lbrycrd API](https://lbry.tech/api/blockchain) (using the lbrycrd daemon API)
## Undocumented API's and other stuff
* Undocumented API (get # views from a file): `https://api.lbry.com/file/view_count?auth_token=abcd&claim_id=wxyz`
* [Special Github repo with all kind of Python scripts](https://github.com/eggplantbren/LBRYnomics/blob/master/fetch_data.py#L137) - like the view count per channel/ summing-up the views per video.
# It is not recommended to modify this file in-place, because it will
# be overwritten during package upgrades. If you want to add further
# options or overwrite existing ones then use
# $ systemctl edit lbrynet.service
# See "man systemd.service" for details.
# /etc/lbry/lbrynet.yml is used for futher settings
[Unit]
Description=LBRYNet
After=network.target
[Service]
ExecStart=/usr/bin/lbrynet start --config /etc/lbry/lbrynet.yml
# Make sure the config directory is readable by the service user
PermissionsStartOnly=true
ExecStartPre=/bin/chgrp lbry /etc/lbry
ExecStartPre=/bin/chgrp lbry /var/lib/lbryum
ExecStartPre=/bin/chgrp lbry /var/lib/lbrynet
# Directory creation and permissions
####################################
# Run as lbry:lbry
User=lbry
Group=lbry
# /etc/lbry
ConfigurationDirectory=lbry
ConfigurationDirectoryMode=0710
# /var/lib/lbrynet & /var/lib/lbryum
StateDirectory=lbrynet lbryum
StateDirectoryMode=0710
# Hardening measures
####################
# Provide a private /tmp and /var/tmp.
PrivateTmp=true
# Mount /usr, /boot/ and /etc read-only for the process.
ProtectSystem=full
# Deny access to /home, /root and /run/user
ProtectHome=true
# Disallow the process and all of its children to gain
# new privileges through execve().
NoNewPrivileges=true
# Use a new /dev namespace only populated with API pseudo devices
# such as /dev/null, /dev/zero and /dev/random.
PrivateDevices=true
# Deny the creation of writable and executable memory mappings.
MemoryDenyWriteExecute=true
[Install]
WantedBy=multi-user.target
server {
listen 443 ssl http2;
server_name your.domain.com;
server_tokens off;
ssl on;
# Certificate & key
ssl_certificate /etc/ssl/your_certificate.crt;
ssl_certificate_key /etc/ssl/your_certificate.key;
ssl_session_timeout 1d;
ssl_session_cache shared:MozSSL:10m; # about 40000 sessions
ssl_session_tickets off;
ssl_dhparam /etc/ssl/certs/dhparam.pem;
# old configuration
ssl_protocols TLSv1 TLSv1.1 TLSv1.2 TLSv1.3;
ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA2$
ssl_prefer_server_ciphers on;
# OCSP stapling
ssl_stapling on;
ssl_stapling_verify on;
location / {
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header Host $http_host;
#proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
# enables WS support
#proxy_http_version 1.1;
#proxy_set_header Upgrade $http_upgrade;
#proxy_set_header Connection "upgrade";
# Lbry bot server runs on port 3009
proxy_pass http://localhost:3009;
}
}
This diff is collapsed.
{
"name": "lbry-channel-feed",
"version": "0.1.0",
"description": "LBRY Channel RSS Feed",
"main": "src/index.js",
"directories": {
"doc": "docs"
},
"scripts": {
"start": "export $(cat tokens.env | xargs) && node .",
"test": "mocha --reporter spec",
"lint": "eslint src --ext .js",
"fix": "eslint src --ext .js --fix"
},
"repository": {
"type": "git",
"url": "git@gitlab.melroy.org:melroy/lbry-channel-feed.git"
},
"keywords": [
"LBRY",
"channel",
"feed",
"RSS",
"provider",
"subscribe"
],
"author": {
"name": "Melroy van den Berg",
"email": "melroy@melroy.org"
},
"license": "AGPL-3.0",
"dependencies": {
"axios": "^0.19.0",
"express": "^4.17.1",
"feed": "^4.0.0",
"qs": "^6.8.0"
},
"devDependencies": {
"chai": "^4.2.0",
"eslint": "^6.1.0",
"eslint-config-standard": "^13.0.1",
"eslint-plugin-import": "^2.18.2",
"eslint-plugin-node": "^9.1.0",
"eslint-plugin-promise": "^4.2.1",
"eslint-plugin-standard": "^4.0.0",
"mocha": "^6.2.0",
"sinon": "^7.4.1"
}
}
// constants
const LBRYNET_HOST = 'localhost'
const LBRYNET_PORT = process.env.LBRYNET_PORT || 5279
const LBRYCRD_HOST = 'localhost'
const LBRYCRD_PORT = process.env.LBRYCRD_PORT || 9245
const LBRYCRD_RPC_USER = 'lbry'
const LBRYCRD_RPC_PASS = process.env.RPC_PASSWORD || 'xyz'
const FEED_URL = 'https://lbryfeed.melroy.org'
const port = process.env.PORT || 3009
const express = require('express')
const routes = require('./routes')
const LBRY = require('./lbry')
const RSSFeed = require('./rssfeed')
// Create helper objects
const lbry = new LBRY(LBRYNET_HOST, LBRYNET_PORT,
LBRYCRD_HOST, LBRYCRD_PORT, LBRYCRD_RPC_USER, LBRYCRD_RPC_PASS)
// Create RSSFeed object
const rssFeed = new RSSFeed(lbry, FEED_URL)
// Create the Express app
const app = express()
app.set('feed', rssFeed)
// Set routes
app.use('/', routes)
// Start server
app.listen(port, () => {
console.log(`LBRY RSS Feed service is listening on ${port}`)
})
This diff is collapsed.
const express = require('express')
const app = express()
app.get('/', (req, res) => {
res.json({ error: 'Provide a channel name. Example: /channel/eevblog' })
})
app.get('/:channel', (req, res) => {
app.get('feed').sendRSSFeed(req.params.channel, res)
})
app.get('/:channel/atom', (req, res) => {
app.get('feed').sendAtomFeed(req.params.channel, res)
})
app.get('/:channel/json', (req, res) => {
app.get('feed').sendJsonFeed(req.params.channel, res)
})
module.exports = app
const express = require('express')
const app = express()
const channelRoute = require('./channel')
app.get('/', (req, res) => {
res.json({ message: 'Welcome to LBRY RSS Feed. Use /channel to follow any channel on LBRY you like!' })
})
.use('/channel', channelRoute)
module.exports = app
const Feed = require('feed').Feed
class RSSFeed {
/**
* Repair defaults
* @param {Object} lbry - LBRY object
* @param {String} URL - Feed URL
*/
constructor (lbry, URL) {
this.feedUrl = URL + '/channel/'
this.lbry = lbry
this.feedOptions = {
description: 'Follow your LBRY channels using your favorite RSS reader!',
link: this.feedUrl,
generator: ' '
}
}
/**
* Retrieve the (RSS) Feed, based on a given channel
* @param {String} channelName - Name of the channel in LBRY
* @return Promose - Return a promise resolve (on the then), when we found items
*/
getFeed (channelName) {
const title = 'LBRY ' + channelName + ' Channel Feed'
const options = this.feedOptions
options.title = title
options.link = this.feedUrl + channelName
options.feedLinks = {
atom: this.feedUrl + channelName + '/atom',
json: this.feedUrl + channelName + '/json'
}
const feed = new Feed(options)
const searchName = '@' + channelName
return this.lbry.search(searchName)
.then(result => {
const items = result.items
// TODO: Add meta data (image, content, description, author), using LBRYNet,LBRYCrd or ChainChannel?
for (let i = 0; i < items.length; i++) {
feed.addItem({
guid: items[i].claim_id,
id: items[i].claim_id,
title: items[i].name,
link: items[i].short_url,
date: new Date(items[i].timestamp * 1000)
})
}
return Promise.resolve(feed)
})
}
/**
* Retrieve Feed in RSS v2.0 format
* @param {String} channelName
* @param {Express Response Object} res