replace node backend with go backend
This commit is contained in:
parent
2cb1f2b554
commit
37083a382f
31 changed files with 902 additions and 4841 deletions
|
|
@ -1,2 +0,0 @@
|
|||
**/*.min.js
|
||||
config.js
|
||||
|
|
@ -1,25 +0,0 @@
|
|||
{
|
||||
"env": {
|
||||
"es6": true,
|
||||
"node": true
|
||||
},
|
||||
"extends": "eslint:recommended",
|
||||
"rules": {
|
||||
"indent": [
|
||||
"error",
|
||||
2
|
||||
],
|
||||
"linebreak-style": [
|
||||
"error",
|
||||
"unix"
|
||||
],
|
||||
"quotes": [
|
||||
"error",
|
||||
"single"
|
||||
],
|
||||
"semi": [
|
||||
"error",
|
||||
"always"
|
||||
]
|
||||
}
|
||||
}
|
||||
47
Dockerfile
47
Dockerfile
|
|
@ -1,27 +1,42 @@
|
|||
FROM node:24-alpine@sha256:7fddd9ddeae8196abf4a3ef2de34e11f7b1a722119f91f28ddf1e99dcafdf114
|
||||
FROM node:24-alpine@sha256:7fddd9ddeae8196abf4a3ef2de34e11f7b1a722119f91f28ddf1e99dcafdf114 AS assets
|
||||
|
||||
RUN mkdir -p /usr/src/app && \
|
||||
chown node:node /usr/src/app
|
||||
WORKDIR /src
|
||||
|
||||
USER node:node
|
||||
COPY package.json package-lock.json ./
|
||||
|
||||
RUN npm install --omit=dev
|
||||
|
||||
FROM golang:1.24-alpine AS build
|
||||
|
||||
WORKDIR /src
|
||||
|
||||
COPY go.mod ./
|
||||
RUN go mod download
|
||||
|
||||
COPY . .
|
||||
COPY --from=assets /src/node_modules ./node_modules
|
||||
|
||||
RUN go build -o /out/haste-server .
|
||||
|
||||
FROM alpine:3.21
|
||||
|
||||
RUN adduser -D -u 1000 appuser
|
||||
|
||||
WORKDIR /usr/src/app
|
||||
|
||||
COPY --chown=node:node package.json package-lock.json ./
|
||||
COPY --from=build /out/haste-server ./haste-server
|
||||
COPY --from=build /src/static ./static
|
||||
COPY --from=build /src/about.md ./about.md
|
||||
COPY --from=build /src/config.js ./config.js
|
||||
COPY --from=build /src/node_modules ./node_modules
|
||||
|
||||
RUN npm install && \
|
||||
chown -R 1000:1000 "/home/node/"
|
||||
USER appuser
|
||||
|
||||
COPY --chown=node:node . .
|
||||
|
||||
ENV STORAGE_TYPE=memcached \
|
||||
ENV STORAGE_TYPE=redis \
|
||||
STORAGE_HOST=127.0.0.1 \
|
||||
STORAGE_PORT=11211 \
|
||||
STORAGE_PORT=6379 \
|
||||
STORAGE_EXPIRE_SECONDS=2592000 \
|
||||
STORAGE_DB=2 \
|
||||
LOGGING_LEVEL=verbose \
|
||||
LOGGING_TYPE=Console \
|
||||
LOGGING_COLORIZE=true \
|
||||
HOST=0.0.0.0 \
|
||||
PORT=7777 \
|
||||
KEY_LENGTH=10 \
|
||||
|
|
@ -39,6 +54,4 @@ ENV STORAGE_TYPE=memcached \
|
|||
|
||||
EXPOSE ${PORT}
|
||||
STOPSIGNAL SIGINT
|
||||
ENTRYPOINT [ "sh", "docker-entrypoint.sh" ]
|
||||
|
||||
CMD ["npm", "start"]
|
||||
CMD ["./haste-server"]
|
||||
|
|
|
|||
2
Procfile
2
Procfile
|
|
@ -1 +1 @@
|
|||
web: node server.js
|
||||
web: ./haste-server
|
||||
|
|
|
|||
218
README.md
218
README.md
|
|
@ -1,8 +1,7 @@
|
|||
# Haste
|
||||
|
||||
Haste is an open-source pastebin software written in node.js, which is easily
|
||||
installable in any network. It can be backed by either redis or filesystem,
|
||||
and has a very easy adapter interface for other stores. A publicly available
|
||||
Haste is an open-source pastebin software written in Go, which is easily
|
||||
installable in any network. It is backed by redis in this version. A publicly available
|
||||
version can be found at [hastebin.com](http://hastebin.com)
|
||||
|
||||
Major design objectives:
|
||||
|
|
@ -30,8 +29,8 @@ STDOUT. Check the README there for more details and usages.
|
|||
|
||||
1. Download the package, and expand it
|
||||
2. Explore the settings inside of config.js, but the defaults should be good
|
||||
3. `npm install`
|
||||
4. `npm start` (you may specify an optional `<config-path>` as well)
|
||||
3. `go build`
|
||||
4. `./haste-server` (you may specify an optional `<config-path>` as well)
|
||||
|
||||
## Settings
|
||||
|
||||
|
|
@ -85,29 +84,8 @@ for the key.
|
|||
|
||||
## Storage
|
||||
|
||||
### File
|
||||
|
||||
To use file storage (the default) change the storage section in `config.js` to
|
||||
something like:
|
||||
|
||||
``` json
|
||||
{
|
||||
"path": "./data",
|
||||
"type": "file"
|
||||
}
|
||||
```
|
||||
|
||||
where `path` represents where you want the files stored.
|
||||
|
||||
File storage currently does not support paste expiration, you can follow [#191](https://github.com/seejohnrun/haste-server/issues/191) for status updates.
|
||||
|
||||
### Redis
|
||||
|
||||
To use redis storage you must install the `redis` package in npm, and have
|
||||
`redis-server` running on the machine.
|
||||
|
||||
`npm install redis`
|
||||
|
||||
Once you've done that, your config section should look like:
|
||||
|
||||
``` json
|
||||
|
|
@ -125,167 +103,9 @@ or post.
|
|||
|
||||
All of which are optional except `type` with very logical default values.
|
||||
|
||||
If your Redis server is configured for password authentification, use the `password` field.
|
||||
If your Redis server is configured for password authentification, use the `password` field. `REDISTOGO_URL` is also supported for URL-based Redis deployments.
|
||||
|
||||
### Postgres
|
||||
|
||||
To use postgres storage you must install the `pg` package in npm
|
||||
|
||||
`npm install pg`
|
||||
|
||||
Once you've done that, your config section should look like:
|
||||
|
||||
``` json
|
||||
{
|
||||
"type": "postgres",
|
||||
"connectionUrl": "postgres://user:password@host:5432/database"
|
||||
}
|
||||
```
|
||||
|
||||
You can also just set the environment variable for `DATABASE_URL` to your database connection url.
|
||||
|
||||
You will have to manually add a table to your postgres database:
|
||||
|
||||
`create table entries (id serial primary key, key varchar(255) not null, value text not null, expiration int, unique(key));`
|
||||
|
||||
You can also set an `expire` option to the number of seconds to expire keys in.
|
||||
This is off by default, but will constantly kick back expirations on each view
|
||||
or post.
|
||||
|
||||
All of which are optional except `type` with very logical default values.
|
||||
|
||||
### MongoDB
|
||||
|
||||
To use mongodb storage you must install the 'mongodb' package in npm
|
||||
|
||||
`npm install mongodb`
|
||||
|
||||
Once you've done that, your config section should look like:
|
||||
|
||||
``` json
|
||||
{
|
||||
"type": "mongodb",
|
||||
"connectionUrl": "mongodb://localhost:27017",
|
||||
"connectionName": "database"
|
||||
}
|
||||
```
|
||||
|
||||
You can also just set the environment variable for `DATABASE_URL` to your server connection url and `DATABASE_NAME` for your database name.
|
||||
|
||||
Unlike with postgres you do NOT have to create the table in your mongo database prior to running.
|
||||
|
||||
You can also set an `expire` option to the number of seconds to expire keys in.
|
||||
This is off by default, but will constantly kick back expirations on each view or post.
|
||||
|
||||
### Memcached
|
||||
|
||||
To use memcache storage you must install the `memcached` package via npm
|
||||
|
||||
`npm install memcached`
|
||||
|
||||
Once you've done that, your config section should look like:
|
||||
|
||||
``` json
|
||||
{
|
||||
"type": "memcached",
|
||||
"host": "127.0.0.1",
|
||||
"port": 11211
|
||||
}
|
||||
```
|
||||
|
||||
You can also set an `expire` option to the number of seconds to expire keys in.
|
||||
This behaves just like the redis expirations, but does not push expirations
|
||||
forward on GETs.
|
||||
|
||||
All of which are optional except `type` with very logical default values.
|
||||
|
||||
### RethinkDB
|
||||
|
||||
To use the RethinkDB storage system, you must install the `rethinkdbdash` package via npm
|
||||
|
||||
`npm install rethinkdbdash`
|
||||
|
||||
Once you've done that, your config section should look like this:
|
||||
|
||||
``` json
|
||||
{
|
||||
"type": "rethinkdb",
|
||||
"host": "127.0.0.1",
|
||||
"port": 28015,
|
||||
"db": "haste"
|
||||
}
|
||||
```
|
||||
|
||||
In order for this to work, the database must be pre-created before the script is ran.
|
||||
Also, you must create an `uploads` table, which will store all the data for uploads.
|
||||
|
||||
You can optionally add the `user` and `password` properties to use a user system.
|
||||
|
||||
### Google Datastore
|
||||
|
||||
To use the Google Datastore storage system, you must install the `@google-cloud/datastore` package via npm
|
||||
|
||||
`npm install @google-cloud/datastore`
|
||||
|
||||
Once you've done that, your config section should look like this:
|
||||
|
||||
``` json
|
||||
{
|
||||
"type": "google-datastore"
|
||||
}
|
||||
```
|
||||
|
||||
Authentication is handled automatically by [Google Cloud service account credentials](https://cloud.google.com/docs/authentication/getting-started), by providing authentication details to the GOOGLE_APPLICATION_CREDENTIALS environmental variable.
|
||||
|
||||
### Amazon S3
|
||||
|
||||
To use [Amazon S3](https://aws.amazon.com/s3/) as a storage system, you must
|
||||
install the `aws-sdk` package via npm:
|
||||
|
||||
`npm install aws-sdk`
|
||||
|
||||
Once you've done that, your config section should look like this:
|
||||
|
||||
```json
|
||||
{
|
||||
"type": "amazon-s3",
|
||||
"bucket": "your-bucket-name",
|
||||
"region": "us-east-1"
|
||||
}
|
||||
```
|
||||
|
||||
Authentication can be handled in the config (by adding `keyId` and `keySecret` properties to the above example), or separately by the client SDK.
|
||||
See [Amazon's documentation](https://docs.aws.amazon.com/sdk-for-javascript/v2/developer-guide/setting-credentials-node.html) for more information.
|
||||
|
||||
You will need to create an IAM user and retrieve an access key. The IAM user should have the the following permissions:
|
||||
|
||||
```json
|
||||
{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [
|
||||
{
|
||||
"Action": [
|
||||
"s3:GetObject",
|
||||
"s3:PutObject"
|
||||
],
|
||||
"Effect": "Allow",
|
||||
"Resource": "arn:aws:s3:::your-bucket-name-goes-here/*"
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
**Alternative S3 API Providers:** A number of other storage providers support the S3 API (such as Linode, BackBlaze B2, etc). To use one of those providers, you will need to specify an endpoint in your config instead of a region:
|
||||
|
||||
```json
|
||||
{
|
||||
"type": "amazon-s3",
|
||||
"bucket": "your-bucket-name",
|
||||
"endpoint": "s3.example.backblazeb2.com",
|
||||
"keyId": "9387200",
|
||||
"keySecret": "H93lmz_93hla9jhdl/Example"
|
||||
}
|
||||
```
|
||||
Only the Redis backend is implemented in this Go version.
|
||||
|
||||
## Docker
|
||||
|
||||
|
|
@ -305,7 +125,7 @@ docker run --name haste-server-container --env STORAGE_TYPE=redis --env STORAGE_
|
|||
|
||||
### Use docker-compose example
|
||||
|
||||
There is an example `docker-compose.yml` which runs haste-server together with memcached
|
||||
There is an example `docker-compose.yml` which runs haste-server together with redis
|
||||
|
||||
```bash
|
||||
docker-compose up
|
||||
|
|
@ -321,24 +141,12 @@ Here is a list of all the environment variables
|
|||
|
||||
| Name | Default value | Description |
|
||||
| :--------------------: | :-----------: | :-----------------------------------------------------------------------------------------------------------: |
|
||||
| STORAGE_TYPE | memcached | Type of storage . Accepted values: "memcached","redis","postgres","rethinkdb", "amazon-s3", and "file" |
|
||||
| STORAGE_HOST | 127.0.0.1 | Storage host. Applicable for types: memcached, redis, postgres, and rethinkdb |
|
||||
| STORAGE_PORT | 11211 | Port on the storage host. Applicable for types: memcached, redis, postgres, and rethinkdb |
|
||||
| STORAGE_EXPIRE_SECONDS | 2592000 | Number of seconds to expire keys in. Applicable for types. Redis, postgres, memcached. `expire` option to the |
|
||||
| STORAGE_DB | 2 | The name of the database. Applicable for redis, postgres, and rethinkdb |
|
||||
| STORAGE_PASSWORD | | Password for database. Applicable for redis, postges, rethinkdb . |
|
||||
| STORAGE_USERNAME | | Database username. Applicable for postgres, and rethinkdb |
|
||||
| STORAGE_AWS_BUCKET | | Applicable for amazon-s3. This is the name of the S3 bucket |
|
||||
| STORAGE_AWS_REGION | | Applicable for amazon-s3. The region in which the bucket is located |
|
||||
| STORAGE_FILEPATH | | Path to file to save data to. Applicable for type file |
|
||||
|
||||
### Logging
|
||||
|
||||
| Name | Default value | Description |
|
||||
| :---------------: | :-----------: | :---------: |
|
||||
| LOGGING_LEVEL | verbose | |
|
||||
| LOGGING_TYPE= | Console |
|
||||
| LOGGING_COLORIZE= | true |
|
||||
| STORAGE_TYPE | redis | Storage type. Only `redis` is supported by this Go backend. |
|
||||
| STORAGE_HOST | 127.0.0.1 | Redis host. |
|
||||
| STORAGE_PORT | 6379 | Redis port. |
|
||||
| STORAGE_EXPIRE_SECONDS | 2592000 | Number of seconds to expire keys in. |
|
||||
| STORAGE_DB | 2 | Redis database index. |
|
||||
| STORAGE_PASSWORD | | Redis password. |
|
||||
|
||||
### Basics
|
||||
|
||||
|
|
|
|||
45
config.js
45
config.js
|
|
@ -1,45 +0,0 @@
|
|||
{
|
||||
|
||||
"host": "0.0.0.0",
|
||||
"port": 7777,
|
||||
|
||||
"keyLength": 10,
|
||||
|
||||
"maxLength": 400000,
|
||||
|
||||
"staticMaxAge": 86400,
|
||||
|
||||
"recompressStaticAssets": true,
|
||||
|
||||
"logging": [
|
||||
{
|
||||
"level": "verbose",
|
||||
"type": "Console",
|
||||
"colorize": true
|
||||
}
|
||||
],
|
||||
|
||||
"keyGenerator": {
|
||||
"type": "phonetic"
|
||||
},
|
||||
|
||||
"rateLimits": {
|
||||
"categories": {
|
||||
"normal": {
|
||||
"totalRequests": 500,
|
||||
"every": 60000
|
||||
}
|
||||
}
|
||||
},
|
||||
|
||||
"storage": {
|
||||
"type": "redis",
|
||||
"host": "localhost",
|
||||
"port": 6379
|
||||
},
|
||||
|
||||
"documents": {
|
||||
"about": "./about.md"
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -1,108 +0,0 @@
|
|||
const {
|
||||
HOST,
|
||||
PORT,
|
||||
KEY_LENGTH,
|
||||
MAX_LENGTH,
|
||||
STATIC_MAX_AGE,
|
||||
RECOMPRESS_STATIC_ASSETS,
|
||||
STORAGE_TYPE,
|
||||
STORAGE_HOST,
|
||||
STORAGE_PORT,
|
||||
STORAGE_EXPIRE_SECONDS,
|
||||
STORAGE_DB,
|
||||
STORAGE_AWS_BUCKET,
|
||||
STORAGE_AWS_REGION,
|
||||
STORAGE_PASSWORD,
|
||||
STORAGE_USERNAME,
|
||||
STORAGE_FILEPATH,
|
||||
LOGGING_LEVEL,
|
||||
LOGGING_TYPE,
|
||||
LOGGING_COLORIZE,
|
||||
KEYGENERATOR_TYPE,
|
||||
KEY_GENERATOR_KEYSPACE,
|
||||
RATE_LIMITS_NORMAL_TOTAL_REQUESTS,
|
||||
RATE_LIMITS_NORMAL_EVERY_MILLISECONDS,
|
||||
RATE_LIMITS_WHITELIST_TOTAL_REQUESTS,
|
||||
RATE_LIMITS_WHITELIST_EVERY_MILLISECONDS,
|
||||
RATE_LIMITS_WHITELIST,
|
||||
RATE_LIMITS_BLACKLIST_TOTAL_REQUESTS,
|
||||
RATE_LIMITS_BLACKLIST_EVERY_MILLISECONDS,
|
||||
RATE_LIMITS_BLACKLIST,
|
||||
DOCUMENTS,
|
||||
} = process.env;
|
||||
|
||||
const config = {
|
||||
host: HOST,
|
||||
port: Number(PORT),
|
||||
|
||||
keyLength: Number(KEY_LENGTH),
|
||||
|
||||
maxLength: Number(MAX_LENGTH),
|
||||
|
||||
staticMaxAge: Number(STATIC_MAX_AGE),
|
||||
|
||||
recompressStaticAssets: RECOMPRESS_STATIC_ASSETS,
|
||||
|
||||
logging: [
|
||||
{
|
||||
level: LOGGING_LEVEL,
|
||||
type: LOGGING_TYPE,
|
||||
colorize: LOGGING_COLORIZE,
|
||||
},
|
||||
],
|
||||
|
||||
keyGenerator: {
|
||||
type: KEYGENERATOR_TYPE,
|
||||
keyspace: KEY_GENERATOR_KEYSPACE,
|
||||
},
|
||||
|
||||
rateLimits: {
|
||||
whitelist: RATE_LIMITS_WHITELIST ? RATE_LIMITS_WHITELIST.split(",") : [],
|
||||
blacklist: RATE_LIMITS_BLACKLIST ? RATE_LIMITS_BLACKLIST.split(",") : [],
|
||||
categories: {
|
||||
normal: {
|
||||
totalRequests: RATE_LIMITS_NORMAL_TOTAL_REQUESTS,
|
||||
every: RATE_LIMITS_NORMAL_EVERY_MILLISECONDS,
|
||||
},
|
||||
whitelist:
|
||||
RATE_LIMITS_WHITELIST_EVERY_MILLISECONDS ||
|
||||
RATE_LIMITS_WHITELIST_TOTAL_REQUESTS
|
||||
? {
|
||||
totalRequests: RATE_LIMITS_WHITELIST_TOTAL_REQUESTS,
|
||||
every: RATE_LIMITS_WHITELIST_EVERY_MILLISECONDS,
|
||||
}
|
||||
: null,
|
||||
blacklist:
|
||||
RATE_LIMITS_BLACKLIST_EVERY_MILLISECONDS ||
|
||||
RATE_LIMITS_BLACKLIST_TOTAL_REQUESTS
|
||||
? {
|
||||
totalRequests: RATE_LIMITS_WHITELIST_TOTAL_REQUESTS,
|
||||
every: RATE_LIMITS_BLACKLIST_EVERY_MILLISECONDS,
|
||||
}
|
||||
: null,
|
||||
},
|
||||
},
|
||||
|
||||
storage: {
|
||||
type: STORAGE_TYPE,
|
||||
host: STORAGE_HOST,
|
||||
port: Number(STORAGE_PORT),
|
||||
expire: Number(STORAGE_EXPIRE_SECONDS),
|
||||
bucket: STORAGE_AWS_BUCKET,
|
||||
region: STORAGE_AWS_REGION,
|
||||
connectionUrl: `postgres://${STORAGE_USERNAME}:${STORAGE_PASSWORD}@${STORAGE_HOST}:${STORAGE_PORT}/${STORAGE_DB}`,
|
||||
db: STORAGE_DB,
|
||||
user: STORAGE_USERNAME,
|
||||
password: STORAGE_PASSWORD,
|
||||
path: STORAGE_FILEPATH,
|
||||
},
|
||||
|
||||
documents: DOCUMENTS
|
||||
? DOCUMENTS.split(",").reduce((acc, item) => {
|
||||
const keyAndValueArray = item.replace(/\s/g, "").split("=");
|
||||
return { ...acc, [keyAndValueArray[0]]: keyAndValueArray[1] };
|
||||
}, {})
|
||||
: null,
|
||||
};
|
||||
|
||||
console.log(JSON.stringify(config));
|
||||
|
|
@ -1,9 +0,0 @@
|
|||
#!/bin/sh
|
||||
|
||||
# We use this file to translate environmental variables to .env files used by the application
|
||||
|
||||
set -e
|
||||
|
||||
node ./docker-entrypoint.js > ./config.js
|
||||
|
||||
exec "$@"
|
||||
10
go.mod
Normal file
10
go.mod
Normal file
|
|
@ -0,0 +1,10 @@
|
|||
module haste-server
|
||||
|
||||
go 1.24.0
|
||||
|
||||
require github.com/redis/go-redis/v9 v9.16.0
|
||||
|
||||
require (
|
||||
github.com/cespare/xxhash/v2 v2.3.0 // indirect
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
|
||||
)
|
||||
10
go.sum
Normal file
10
go.sum
Normal file
|
|
@ -0,0 +1,10 @@
|
|||
github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=
|
||||
github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c=
|
||||
github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA=
|
||||
github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0=
|
||||
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
|
||||
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
|
||||
github.com/redis/go-redis/v9 v9.16.0 h1:OotgqgLSRCmzfqChbQyG1PHC3tLNR89DG4jdOERSEP4=
|
||||
github.com/redis/go-redis/v9 v9.16.0/go.mod h1:u410H11HMLoB+TP67dz8rL9s6QW2j76l0//kSOd3370=
|
||||
|
|
@ -1,155 +0,0 @@
|
|||
var winston = require('winston');
|
||||
var Busboy = require('busboy');
|
||||
|
||||
// For handling serving stored documents
|
||||
|
||||
var DocumentHandler = function(options) {
|
||||
if (!options) {
|
||||
options = {};
|
||||
}
|
||||
this.keyLength = options.keyLength || DocumentHandler.defaultKeyLength;
|
||||
this.maxLength = options.maxLength; // none by default
|
||||
this.store = options.store;
|
||||
this.keyGenerator = options.keyGenerator;
|
||||
};
|
||||
|
||||
DocumentHandler.defaultKeyLength = 10;
|
||||
|
||||
// Handle retrieving a document
|
||||
DocumentHandler.prototype.handleGet = function(request, response, config) {
|
||||
const key = request.params.id.split('.')[0];
|
||||
const skipExpire = !!config.documents[key];
|
||||
|
||||
this.store.get(key, function(ret) {
|
||||
if (ret) {
|
||||
winston.verbose('retrieved document', { key: key });
|
||||
response.writeHead(200, { 'content-type': 'application/json' });
|
||||
if (request.method === 'HEAD') {
|
||||
response.end();
|
||||
} else {
|
||||
response.end(JSON.stringify({ data: ret, key: key }));
|
||||
}
|
||||
}
|
||||
else {
|
||||
winston.warn('document not found', { key: key });
|
||||
response.writeHead(404, { 'content-type': 'application/json' });
|
||||
if (request.method === 'HEAD') {
|
||||
response.end();
|
||||
} else {
|
||||
response.end(JSON.stringify({ message: 'Document not found.' }));
|
||||
}
|
||||
}
|
||||
}, skipExpire);
|
||||
};
|
||||
|
||||
// Handle retrieving the raw version of a document
|
||||
DocumentHandler.prototype.handleRawGet = function(request, response, config) {
|
||||
const key = request.params.id.split('.')[0];
|
||||
const skipExpire = !!config.documents[key];
|
||||
|
||||
this.store.get(key, function(ret) {
|
||||
if (ret) {
|
||||
winston.verbose('retrieved raw document', { key: key });
|
||||
response.writeHead(200, { 'content-type': 'text/plain; charset=UTF-8' });
|
||||
if (request.method === 'HEAD') {
|
||||
response.end();
|
||||
} else {
|
||||
response.end(ret);
|
||||
}
|
||||
}
|
||||
else {
|
||||
winston.warn('raw document not found', { key: key });
|
||||
response.writeHead(404, { 'content-type': 'application/json' });
|
||||
if (request.method === 'HEAD') {
|
||||
response.end();
|
||||
} else {
|
||||
response.end(JSON.stringify({ message: 'Document not found.' }));
|
||||
}
|
||||
}
|
||||
}, skipExpire);
|
||||
};
|
||||
|
||||
// Handle adding a new Document
|
||||
DocumentHandler.prototype.handlePost = function (request, response) {
|
||||
var _this = this;
|
||||
var buffer = '';
|
||||
var cancelled = false;
|
||||
|
||||
// What to do when done
|
||||
var onSuccess = function () {
|
||||
// Check length
|
||||
if (_this.maxLength && buffer.length > _this.maxLength) {
|
||||
cancelled = true;
|
||||
winston.warn('document >maxLength', { maxLength: _this.maxLength });
|
||||
response.writeHead(400, { 'content-type': 'application/json' });
|
||||
response.end(
|
||||
JSON.stringify({ message: 'Document exceeds maximum length.' })
|
||||
);
|
||||
return;
|
||||
}
|
||||
// And then save if we should
|
||||
_this.chooseKey(function (key) {
|
||||
_this.store.set(key, buffer, function (res) {
|
||||
if (res) {
|
||||
winston.verbose('added document', { key: key });
|
||||
response.writeHead(200, { 'content-type': 'application/json' });
|
||||
response.end(JSON.stringify({ key: key }));
|
||||
}
|
||||
else {
|
||||
winston.verbose('error adding document');
|
||||
response.writeHead(500, { 'content-type': 'application/json' });
|
||||
response.end(JSON.stringify({ message: 'Error adding document.' }));
|
||||
}
|
||||
});
|
||||
});
|
||||
};
|
||||
|
||||
// If we should, parse a form to grab the data
|
||||
var ct = request.headers['content-type'];
|
||||
if (ct && ct.split(';')[0] === 'multipart/form-data') {
|
||||
var busboy = new Busboy({ headers: request.headers });
|
||||
busboy.on('field', function (fieldname, val) {
|
||||
if (fieldname === 'data') {
|
||||
buffer = val;
|
||||
}
|
||||
});
|
||||
busboy.on('finish', function () {
|
||||
onSuccess();
|
||||
});
|
||||
request.pipe(busboy);
|
||||
// Otherwise, use our own and just grab flat data from POST body
|
||||
} else {
|
||||
request.on('data', function (data) {
|
||||
buffer += data.toString();
|
||||
});
|
||||
request.on('end', function () {
|
||||
if (cancelled) { return; }
|
||||
onSuccess();
|
||||
});
|
||||
request.on('error', function (error) {
|
||||
winston.error('connection error: ' + error.message);
|
||||
response.writeHead(500, { 'content-type': 'application/json' });
|
||||
response.end(JSON.stringify({ message: 'Connection error.' }));
|
||||
cancelled = true;
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
// Keep choosing keys until one isn't taken
|
||||
DocumentHandler.prototype.chooseKey = function(callback) {
|
||||
var key = this.acceptableKey();
|
||||
var _this = this;
|
||||
this.store.get(key, function(ret) {
|
||||
if (ret) {
|
||||
_this.chooseKey(callback);
|
||||
} else {
|
||||
callback(key);
|
||||
}
|
||||
}, true); // Don't bump expirations when key searching
|
||||
};
|
||||
|
||||
DocumentHandler.prototype.acceptableKey = function() {
|
||||
return this.keyGenerator.createKey(this.keyLength);
|
||||
};
|
||||
|
||||
module.exports = DocumentHandler;
|
||||
|
|
@ -1,65 +0,0 @@
|
|||
/*global require,module,process*/
|
||||
|
||||
var AWS = require('aws-sdk');
|
||||
var winston = require('winston');
|
||||
|
||||
var AmazonS3DocumentStore = function(options) {
|
||||
this.expire = options.expire;
|
||||
this.bucket = options.bucket;
|
||||
|
||||
var config = {};
|
||||
if(options.endpoint)
|
||||
config.endpoint = new AWS.Endpoint(options.endpoint);
|
||||
if(options.region)
|
||||
config.region = options.region;
|
||||
if(options.keyId && options.keySecret)
|
||||
config.credentials = new AWS.Credentials({ accessKeyId: options.keyId, secretAccessKey: options.keySecret })
|
||||
|
||||
this.client = new AWS.S3(config);
|
||||
};
|
||||
|
||||
AmazonS3DocumentStore.prototype.get = function(key, callback, skipExpire) {
|
||||
var _this = this;
|
||||
|
||||
var req = {
|
||||
Bucket: _this.bucket,
|
||||
Key: key
|
||||
};
|
||||
|
||||
_this.client.getObject(req, function(err, data) {
|
||||
if(err) {
|
||||
callback(false);
|
||||
}
|
||||
else {
|
||||
callback(data.Body.toString('utf-8'));
|
||||
if (_this.expire && !skipExpire) {
|
||||
winston.warn('amazon s3 store cannot set expirations on keys');
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
AmazonS3DocumentStore.prototype.set = function(key, data, callback, skipExpire) {
|
||||
var _this = this;
|
||||
|
||||
var req = {
|
||||
Bucket: _this.bucket,
|
||||
Key: key,
|
||||
Body: data,
|
||||
ContentType: 'text/plain'
|
||||
};
|
||||
|
||||
_this.client.putObject(req, function(err, data) {
|
||||
if (err) {
|
||||
callback(false);
|
||||
}
|
||||
else {
|
||||
callback(true);
|
||||
if (_this.expire && !skipExpire) {
|
||||
winston.warn('amazon s3 store cannot set expirations on keys');
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
module.exports = AmazonS3DocumentStore;
|
||||
|
|
@ -1,63 +0,0 @@
|
|||
var fs = require('fs');
|
||||
var crypto = require('crypto');
|
||||
|
||||
var winston = require('winston');
|
||||
|
||||
// For storing in files
|
||||
// options[type] = file
|
||||
// options[path] - Where to store
|
||||
|
||||
var FileDocumentStore = function(options) {
|
||||
this.basePath = options.path || './data';
|
||||
this.expire = options.expire;
|
||||
};
|
||||
|
||||
// Generate md5 of a string
|
||||
FileDocumentStore.md5 = function(str) {
|
||||
var md5sum = crypto.createHash('md5');
|
||||
md5sum.update(str);
|
||||
return md5sum.digest('hex');
|
||||
};
|
||||
|
||||
// Save data in a file, key as md5 - since we don't know what we could
|
||||
// be passed here
|
||||
FileDocumentStore.prototype.set = function(key, data, callback, skipExpire) {
|
||||
try {
|
||||
var _this = this;
|
||||
fs.mkdir(this.basePath, '700', function() {
|
||||
var fn = _this.basePath + '/' + FileDocumentStore.md5(key);
|
||||
fs.writeFile(fn, data, 'utf8', function(err) {
|
||||
if (err) {
|
||||
callback(false);
|
||||
}
|
||||
else {
|
||||
callback(true);
|
||||
if (_this.expire && !skipExpire) {
|
||||
winston.warn('file store cannot set expirations on keys');
|
||||
}
|
||||
}
|
||||
});
|
||||
});
|
||||
} catch(err) {
|
||||
callback(false);
|
||||
}
|
||||
};
|
||||
|
||||
// Get data from a file from key
|
||||
FileDocumentStore.prototype.get = function(key, callback, skipExpire) {
|
||||
var _this = this;
|
||||
var fn = this.basePath + '/' + FileDocumentStore.md5(key);
|
||||
fs.readFile(fn, 'utf8', function(err, data) {
|
||||
if (err) {
|
||||
callback(false);
|
||||
}
|
||||
else {
|
||||
callback(data);
|
||||
if (_this.expire && !skipExpire) {
|
||||
winston.warn('file store cannot set expirations on keys');
|
||||
}
|
||||
}
|
||||
});
|
||||
};
|
||||
|
||||
module.exports = FileDocumentStore;
|
||||
|
|
@ -1,89 +0,0 @@
|
|||
/*global require,module,process*/
|
||||
|
||||
const Datastore = require('@google-cloud/datastore');
|
||||
const winston = require('winston');
|
||||
|
||||
class GoogleDatastoreDocumentStore {
|
||||
|
||||
// Create a new store with options
|
||||
constructor(options) {
|
||||
this.kind = "Haste";
|
||||
this.expire = options.expire;
|
||||
this.datastore = new Datastore();
|
||||
}
|
||||
|
||||
// Save file in a key
|
||||
set(key, data, callback, skipExpire) {
|
||||
var expireTime = (skipExpire || this.expire === undefined) ? null : new Date(Date.now() + this.expire * 1000);
|
||||
|
||||
var taskKey = this.datastore.key([this.kind, key])
|
||||
var task = {
|
||||
key: taskKey,
|
||||
data: [
|
||||
{
|
||||
name: 'value',
|
||||
value: data,
|
||||
excludeFromIndexes: true
|
||||
},
|
||||
{
|
||||
name: 'expiration',
|
||||
value: expireTime
|
||||
}
|
||||
]
|
||||
};
|
||||
|
||||
this.datastore.insert(task).then(() => {
|
||||
callback(true);
|
||||
})
|
||||
.catch(err => {
|
||||
callback(false);
|
||||
});
|
||||
}
|
||||
|
||||
// Get a file from a key
|
||||
get(key, callback, skipExpire) {
|
||||
var taskKey = this.datastore.key([this.kind, key])
|
||||
|
||||
this.datastore.get(taskKey).then((entity) => {
|
||||
if (skipExpire || entity[0]["expiration"] == null) {
|
||||
callback(entity[0]["value"]);
|
||||
}
|
||||
else {
|
||||
// check for expiry
|
||||
if (entity[0]["expiration"] < new Date()) {
|
||||
winston.info("document expired", {key: key, expiration: entity[0]["expiration"], check: new Date(null)});
|
||||
callback(false);
|
||||
}
|
||||
else {
|
||||
// update expiry
|
||||
var task = {
|
||||
key: taskKey,
|
||||
data: [
|
||||
{
|
||||
name: 'value',
|
||||
value: entity[0]["value"],
|
||||
excludeFromIndexes: true
|
||||
},
|
||||
{
|
||||
name: 'expiration',
|
||||
value: new Date(Date.now() + this.expire * 1000)
|
||||
}
|
||||
]
|
||||
};
|
||||
this.datastore.update(task).then(() => {
|
||||
})
|
||||
.catch(err => {
|
||||
winston.error("failed to update expiration", {error: err});
|
||||
});
|
||||
callback(entity[0]["value"]);
|
||||
}
|
||||
}
|
||||
})
|
||||
.catch(err => {
|
||||
winston.error("Error retrieving value from Google Datastore", {error: err});
|
||||
callback(false);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = GoogleDatastoreDocumentStore;
|
||||
|
|
@ -1,54 +0,0 @@
|
|||
const memcached = require('memcached');
|
||||
const winston = require('winston');
|
||||
|
||||
class MemcachedDocumentStore {
|
||||
|
||||
// Create a new store with options
|
||||
constructor(options) {
|
||||
this.expire = options.expire;
|
||||
|
||||
const host = options.host || '127.0.0.1';
|
||||
const port = options.port || 11211;
|
||||
const url = `${host}:${port}`;
|
||||
this.connect(url);
|
||||
}
|
||||
|
||||
// Create a connection
|
||||
connect(url) {
|
||||
this.client = new memcached(url);
|
||||
|
||||
winston.info(`connecting to memcached on ${url}`);
|
||||
|
||||
this.client.on('failure', function(error) {
|
||||
winston.info('error connecting to memcached', {error});
|
||||
});
|
||||
}
|
||||
|
||||
// Save file in a key
|
||||
set(key, data, callback, skipExpire) {
|
||||
this.client.set(key, data, skipExpire ? 0 : this.expire || 0, (error) => {
|
||||
callback(!error);
|
||||
});
|
||||
}
|
||||
|
||||
// Get a file from a key
|
||||
get(key, callback, skipExpire) {
|
||||
this.client.get(key, (error, data) => {
|
||||
const value = error ? false : data;
|
||||
|
||||
callback(value);
|
||||
|
||||
// Update the key so that the expiration is pushed forward
|
||||
if (value && !skipExpire) {
|
||||
this.set(key, data, (updateSucceeded) => {
|
||||
if (!updateSucceeded) {
|
||||
winston.error('failed to update expiration on GET', {key});
|
||||
}
|
||||
}, skipExpire);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
module.exports = MemcachedDocumentStore;
|
||||
|
|
@ -1,110 +0,0 @@
|
|||
const { MongoClient } = require("mongodb"),
|
||||
winston = require("winston");
|
||||
|
||||
const MongoDocumentStore = function (options) {
|
||||
this.expire = options.expire;
|
||||
this.connectionUrl = process.env.DATABASE_URL || options.connectionUrl;
|
||||
this.connectionName = process.env.DATABASE_NAME || options.connectionName;
|
||||
};
|
||||
|
||||
MongoDocumentStore.prototype.set = function (key, data, callback, skipExpire) {
|
||||
const now = Math.floor(new Date().getTime() / 1000);
|
||||
|
||||
this.safeConnect((err, db, client) => {
|
||||
if (err) return callback(false);
|
||||
|
||||
db.collection("entries").updateOne(
|
||||
{
|
||||
entry_id: key,
|
||||
$or: [{ expiration: -1 }, { expiration: { $gt: now } }],
|
||||
},
|
||||
{
|
||||
$set: {
|
||||
entry_id: key,
|
||||
value: data,
|
||||
expiration:
|
||||
this.expire && !skipExpire ? this.expire + now : -1,
|
||||
},
|
||||
},
|
||||
{
|
||||
upsert: true,
|
||||
},
|
||||
(err /*, existing*/) => {
|
||||
client.close();
|
||||
|
||||
if (err) {
|
||||
winston.error("error persisting value to mongodb", {
|
||||
error: err,
|
||||
});
|
||||
return callback(false);
|
||||
}
|
||||
|
||||
callback(true);
|
||||
}
|
||||
);
|
||||
});
|
||||
};
|
||||
|
||||
MongoDocumentStore.prototype.get = function (key, callback, skipExpire) {
|
||||
const now = Math.floor(new Date().getTime() / 1000);
|
||||
|
||||
this.safeConnect((err, db, client) => {
|
||||
if (err) return callback(false);
|
||||
|
||||
db.collection("entries").findOne(
|
||||
{
|
||||
entry_id: key,
|
||||
$or: [{ expiration: -1 }, { expiration: { $gt: now } }],
|
||||
},
|
||||
(err, entry) => {
|
||||
if (err) {
|
||||
winston.error("error persisting value to mongodb", {
|
||||
error: err,
|
||||
});
|
||||
client.close();
|
||||
return callback(false);
|
||||
}
|
||||
|
||||
callback(entry === null ? false : entry.value);
|
||||
|
||||
if (
|
||||
entry !== null &&
|
||||
entry.expiration !== -1 &&
|
||||
this.expire &&
|
||||
!skipExpire
|
||||
) {
|
||||
db.collection("entries").updateOne(
|
||||
{
|
||||
entry_id: key,
|
||||
},
|
||||
{
|
||||
$set: {
|
||||
expiration: this.expire + now,
|
||||
},
|
||||
},
|
||||
() => {
|
||||
client.close();
|
||||
}
|
||||
);
|
||||
}
|
||||
}
|
||||
);
|
||||
});
|
||||
};
|
||||
|
||||
MongoDocumentStore.prototype.safeConnect = function (callback) {
|
||||
MongoClient.connect(
|
||||
this.connectionUrl,
|
||||
{ useUnifiedTopology: true },
|
||||
(err, client) => {
|
||||
if (err) {
|
||||
winston.error("error connecting to mongodb", { error: err });
|
||||
callback(err);
|
||||
} else {
|
||||
callback(undefined, client.db(this.connectionName), client);
|
||||
}
|
||||
}
|
||||
);
|
||||
};
|
||||
|
||||
module.exports = MongoDocumentStore;
|
||||
|
|
@ -1,80 +0,0 @@
|
|||
/*global require,module,process*/
|
||||
|
||||
var winston = require('winston');
|
||||
const {Pool} = require('pg');
|
||||
|
||||
// create table entries (id serial primary key, key varchar(255) not null, value text not null, expiration int, unique(key));
|
||||
|
||||
// A postgres document store
|
||||
var PostgresDocumentStore = function (options) {
|
||||
this.expireJS = parseInt(options.expire, 10);
|
||||
|
||||
const connectionString = process.env.DATABASE_URL || options.connectionUrl;
|
||||
this.pool = new Pool({connectionString});
|
||||
};
|
||||
|
||||
PostgresDocumentStore.prototype = {
|
||||
|
||||
// Set a given key
|
||||
set: function (key, data, callback, skipExpire) {
|
||||
var now = Math.floor(new Date().getTime() / 1000);
|
||||
var that = this;
|
||||
this.safeConnect(function (err, client, done) {
|
||||
if (err) { return callback(false); }
|
||||
client.query('INSERT INTO entries (key, value, expiration) VALUES ($1, $2, $3)', [
|
||||
key,
|
||||
data,
|
||||
that.expireJS && !skipExpire ? that.expireJS + now : null
|
||||
], function (err) {
|
||||
if (err) {
|
||||
winston.error('error persisting value to postgres', { error: err });
|
||||
return callback(false);
|
||||
}
|
||||
callback(true);
|
||||
done();
|
||||
});
|
||||
});
|
||||
},
|
||||
|
||||
// Get a given key's data
|
||||
get: function (key, callback, skipExpire) {
|
||||
var now = Math.floor(new Date().getTime() / 1000);
|
||||
var that = this;
|
||||
this.safeConnect(function (err, client, done) {
|
||||
if (err) { return callback(false); }
|
||||
client.query('SELECT id,value,expiration from entries where KEY = $1 and (expiration IS NULL or expiration > $2)', [key, now], function (err, result) {
|
||||
if (err) {
|
||||
winston.error('error retrieving value from postgres', { error: err });
|
||||
return callback(false);
|
||||
}
|
||||
callback(result.rows.length ? result.rows[0].value : false);
|
||||
if (result.rows.length && that.expireJS && !skipExpire) {
|
||||
client.query('UPDATE entries SET expiration = $1 WHERE ID = $2', [
|
||||
that.expireJS + now,
|
||||
result.rows[0].id
|
||||
], function (err) {
|
||||
if (!err) {
|
||||
done();
|
||||
}
|
||||
});
|
||||
} else {
|
||||
done();
|
||||
}
|
||||
});
|
||||
});
|
||||
},
|
||||
|
||||
// A connection wrapper
|
||||
safeConnect: function (callback) {
|
||||
this.pool.connect((error, client, done) => {
|
||||
if (error) {
|
||||
winston.error('error connecting to postgres', {error});
|
||||
callback(error);
|
||||
} else {
|
||||
callback(undefined, client, done);
|
||||
}
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
module.exports = PostgresDocumentStore;
|
||||
|
|
@ -1,83 +0,0 @@
|
|||
const { createClient } = require('redis');
|
||||
const winston = require('winston');
|
||||
|
||||
class RedisDocumentStore {
|
||||
constructor(options, client) {
|
||||
this.expire = options.expire;
|
||||
if (client) {
|
||||
winston.info('using predefined redis client');
|
||||
RedisDocumentStore.client = client;
|
||||
}
|
||||
}
|
||||
|
||||
static async connect(options) {
|
||||
const host = options.host || '127.0.0.1';
|
||||
const port = options.port || 6379;
|
||||
const index = options.db || 0;
|
||||
|
||||
const client = createClient({
|
||||
socket: {
|
||||
host,
|
||||
port
|
||||
},
|
||||
password: options.password || undefined,
|
||||
database: index
|
||||
});
|
||||
|
||||
client.on('error', (err) => {
|
||||
winston.error('redis disconnected', err);
|
||||
});
|
||||
|
||||
try {
|
||||
await client.connect();
|
||||
winston.info(`connected to redis on ${host}:${port}/${index}`);
|
||||
RedisDocumentStore.client = client;
|
||||
} catch (err) {
|
||||
winston.error('error connecting to redis', { error: err });
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
static async create(options, client) {
|
||||
const store = new RedisDocumentStore(options, client);
|
||||
if (!client && !RedisDocumentStore.client) {
|
||||
winston.info('configuring redis');
|
||||
await RedisDocumentStore.connect(options);
|
||||
}
|
||||
return store;
|
||||
}
|
||||
|
||||
async set(key, data, callback, skipExpire) {
|
||||
try {
|
||||
await RedisDocumentStore.client.set(key, data);
|
||||
if (!skipExpire) {
|
||||
this.setExpiration(key);
|
||||
}
|
||||
callback(true);
|
||||
} catch (err) {
|
||||
callback(false);
|
||||
}
|
||||
}
|
||||
|
||||
setExpiration(key) {
|
||||
if (this.expire) {
|
||||
RedisDocumentStore.client.expire(key, this.expire).catch((err) => {
|
||||
winston.error('failed to set expiry on key: ' + key, err);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
async get(key, callback, skipExpire) {
|
||||
try {
|
||||
const reply = await RedisDocumentStore.client.get(key);
|
||||
if (!skipExpire) {
|
||||
this.setExpiration(key);
|
||||
}
|
||||
callback(reply);
|
||||
} catch (err) {
|
||||
callback(false);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = RedisDocumentStore;
|
||||
|
|
@ -1,46 +0,0 @@
|
|||
const crypto = require('crypto');
|
||||
const rethink = require('rethinkdbdash');
|
||||
const winston = require('winston');
|
||||
|
||||
const md5 = (str) => {
|
||||
const md5sum = crypto.createHash('md5');
|
||||
md5sum.update(str);
|
||||
return md5sum.digest('hex');
|
||||
};
|
||||
|
||||
class RethinkDBStore {
|
||||
constructor(options) {
|
||||
this.client = rethink({
|
||||
silent: true,
|
||||
host: options.host || '127.0.0.1',
|
||||
port: options.port || 28015,
|
||||
db: options.db || 'haste',
|
||||
user: options.user || 'admin',
|
||||
password: options.password || ''
|
||||
});
|
||||
}
|
||||
|
||||
set(key, data, callback) {
|
||||
this.client.table('uploads').insert({ id: md5(key), data: data }).run((error) => {
|
||||
if (error) {
|
||||
callback(false);
|
||||
winston.error('failed to insert to table', error);
|
||||
return;
|
||||
}
|
||||
callback(true);
|
||||
});
|
||||
}
|
||||
|
||||
get(key, callback) {
|
||||
this.client.table('uploads').get(md5(key)).run((error, result) => {
|
||||
if (error || !result) {
|
||||
callback(false);
|
||||
if (error) winston.error('failed to insert to table', error);
|
||||
return;
|
||||
}
|
||||
callback(result.data);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = RethinkDBStore;
|
||||
|
|
@ -1,32 +0,0 @@
|
|||
const fs = require('fs');
|
||||
|
||||
module.exports = class DictionaryGenerator {
|
||||
|
||||
constructor(options, readyCallback) {
|
||||
// Check options format
|
||||
if (!options) throw Error('No options passed to generator');
|
||||
if (!options.path) throw Error('No dictionary path specified in options');
|
||||
|
||||
// Load dictionary
|
||||
fs.readFile(options.path, 'utf8', (err, data) => {
|
||||
if (err) throw err;
|
||||
|
||||
this.dictionary = data.split(/[\n\r]+/);
|
||||
|
||||
if (readyCallback) readyCallback();
|
||||
});
|
||||
}
|
||||
|
||||
// Generates a dictionary-based key, of keyLength words
|
||||
createKey(keyLength) {
|
||||
let text = '';
|
||||
|
||||
for (let i = 0; i < keyLength; i++) {
|
||||
const index = Math.floor(Math.random() * this.dictionary.length);
|
||||
text += this.dictionary[index];
|
||||
}
|
||||
|
||||
return text;
|
||||
}
|
||||
|
||||
};
|
||||
|
|
@ -1,27 +0,0 @@
|
|||
// Draws inspiration from pwgen and http://tools.arantius.com/password
|
||||
|
||||
const randOf = (collection) => {
|
||||
return () => {
|
||||
return collection[Math.floor(Math.random() * collection.length)];
|
||||
};
|
||||
};
|
||||
|
||||
// Helper methods to get an random vowel or consonant
|
||||
const randVowel = randOf('aeiou');
|
||||
const randConsonant = randOf('bcdfghjklmnpqrstvwxyz');
|
||||
|
||||
module.exports = class PhoneticKeyGenerator {
|
||||
|
||||
// Generate a phonetic key of alternating consonant & vowel
|
||||
createKey(keyLength) {
|
||||
let text = '';
|
||||
const start = Math.round(Math.random());
|
||||
|
||||
for (let i = 0; i < keyLength; i++) {
|
||||
text += (i % 2 == start) ? randConsonant() : randVowel();
|
||||
}
|
||||
|
||||
return text;
|
||||
}
|
||||
|
||||
};
|
||||
|
|
@ -1,20 +0,0 @@
|
|||
module.exports = class RandomKeyGenerator {
|
||||
|
||||
// Initialize a new generator with the given keySpace
|
||||
constructor(options = {}) {
|
||||
this.keyspace = options.keyspace || 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789';
|
||||
}
|
||||
|
||||
// Generate a key of the given length
|
||||
createKey(keyLength) {
|
||||
var text = '';
|
||||
|
||||
for (var i = 0; i < keyLength; i++) {
|
||||
const index = Math.floor(Math.random() * this.keyspace.length);
|
||||
text += this.keyspace.charAt(index);
|
||||
}
|
||||
|
||||
return text;
|
||||
}
|
||||
|
||||
};
|
||||
827
main.go
Normal file
827
main.go
Normal file
|
|
@ -0,0 +1,827 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"embed"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
mathrand "math/rand"
|
||||
"mime"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/redis/go-redis/v9"
|
||||
)
|
||||
|
||||
//go:embed static/*
|
||||
var embeddedStatic embed.FS
|
||||
|
||||
type Config struct {
|
||||
Host string `json:"host"`
|
||||
Port int `json:"port"`
|
||||
KeyLength int `json:"keyLength"`
|
||||
MaxLength int `json:"maxLength"`
|
||||
StaticMaxAge int `json:"staticMaxAge"`
|
||||
RecompressStaticAssets bool `json:"recompressStaticAssets"`
|
||||
Logging []LoggingConfig `json:"logging"`
|
||||
KeyGenerator KeyGeneratorConfig `json:"keyGenerator"`
|
||||
RateLimits RateLimitsConfig `json:"rateLimits"`
|
||||
Storage StorageConfig `json:"storage"`
|
||||
Documents map[string]string `json:"documents"`
|
||||
}
|
||||
|
||||
type LoggingConfig struct {
|
||||
Level string `json:"level"`
|
||||
Type string `json:"type"`
|
||||
Colorize any `json:"colorize"`
|
||||
}
|
||||
|
||||
type KeyGeneratorConfig struct {
|
||||
Type string `json:"type"`
|
||||
Keyspace string `json:"keyspace"`
|
||||
Path string `json:"path"`
|
||||
}
|
||||
|
||||
type RateLimitsConfig struct {
|
||||
Whitelist []string `json:"whitelist"`
|
||||
Blacklist []string `json:"blacklist"`
|
||||
Categories map[string]RateLimitBucket `json:"categories"`
|
||||
}
|
||||
|
||||
type RateLimitBucket struct {
|
||||
TotalRequests int `json:"totalRequests"`
|
||||
Every int `json:"every"`
|
||||
}
|
||||
|
||||
type StorageConfig struct {
|
||||
Type string `json:"type"`
|
||||
Host string `json:"host"`
|
||||
Port int `json:"port"`
|
||||
Expire int `json:"expire"`
|
||||
DB int `json:"db"`
|
||||
Password string `json:"password"`
|
||||
}
|
||||
|
||||
type DocumentStore interface {
|
||||
Get(ctx context.Context, key string, skipExpire bool) (string, bool, error)
|
||||
Set(ctx context.Context, key, value string, skipExpire bool) error
|
||||
}
|
||||
|
||||
type RedisStore struct {
|
||||
client *redis.Client
|
||||
expire time.Duration
|
||||
}
|
||||
|
||||
func NewRedisStore(cfg StorageConfig) (*RedisStore, error) {
|
||||
if rawURL := os.Getenv("REDISTOGO_URL"); rawURL != "" {
|
||||
opts, err := redis.ParseURL(rawURL)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
client := redis.NewClient(opts)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
if err := client.Ping(ctx).Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
store := &RedisStore{
|
||||
client: client,
|
||||
expire: time.Duration(cfg.Expire) * time.Second,
|
||||
}
|
||||
log.Printf("connected to redis via REDISTOGO_URL")
|
||||
return store, nil
|
||||
}
|
||||
|
||||
host := cfg.Host
|
||||
if host == "" {
|
||||
host = "127.0.0.1"
|
||||
}
|
||||
port := cfg.Port
|
||||
if port == 0 {
|
||||
port = 6379
|
||||
}
|
||||
|
||||
client := redis.NewClient(&redis.Options{
|
||||
Addr: fmt.Sprintf("%s:%d", host, port),
|
||||
Password: cfg.Password,
|
||||
DB: cfg.DB,
|
||||
})
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
if err := client.Ping(ctx).Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
store := &RedisStore{
|
||||
client: client,
|
||||
expire: time.Duration(cfg.Expire) * time.Second,
|
||||
}
|
||||
log.Printf("connected to redis on %s:%d/%d", host, port, cfg.DB)
|
||||
return store, nil
|
||||
}
|
||||
|
||||
func (s *RedisStore) Get(ctx context.Context, key string, skipExpire bool) (string, bool, error) {
|
||||
val, err := s.client.Get(ctx, key).Result()
|
||||
if errors.Is(err, redis.Nil) {
|
||||
return "", false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return "", false, err
|
||||
}
|
||||
if !skipExpire && s.expire > 0 {
|
||||
go s.client.Expire(context.Background(), key, s.expire)
|
||||
}
|
||||
return val, true, nil
|
||||
}
|
||||
|
||||
func (s *RedisStore) Set(ctx context.Context, key, value string, skipExpire bool) error {
|
||||
if err := s.client.Set(ctx, key, value, 0).Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
if !skipExpire && s.expire > 0 {
|
||||
go s.client.Expire(context.Background(), key, s.expire)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type KeyGenerator interface {
|
||||
CreateKey(length int) string
|
||||
}
|
||||
|
||||
type RandomKeyGenerator struct {
|
||||
keyspace string
|
||||
}
|
||||
|
||||
func (g RandomKeyGenerator) CreateKey(length int) string {
|
||||
if g.keyspace == "" {
|
||||
g.keyspace = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
|
||||
}
|
||||
var b strings.Builder
|
||||
for i := 0; i < length; i++ {
|
||||
b.WriteByte(g.keyspace[mathrand.Intn(len(g.keyspace))])
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
type PhoneticKeyGenerator struct{}
|
||||
|
||||
func (g PhoneticKeyGenerator) CreateKey(length int) string {
|
||||
const vowels = "aeiou"
|
||||
const consonants = "bcdfghjklmnpqrstvwxyz"
|
||||
start := mathrand.Intn(2)
|
||||
var b strings.Builder
|
||||
for i := 0; i < length; i++ {
|
||||
if i%2 == start {
|
||||
b.WriteByte(consonants[mathrand.Intn(len(consonants))])
|
||||
continue
|
||||
}
|
||||
b.WriteByte(vowels[mathrand.Intn(len(vowels))])
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
type DictionaryKeyGenerator struct {
|
||||
words []string
|
||||
}
|
||||
|
||||
func (g DictionaryKeyGenerator) CreateKey(length int) string {
|
||||
var b strings.Builder
|
||||
for i := 0; i < length; i++ {
|
||||
b.WriteString(g.words[mathrand.Intn(len(g.words))])
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
type Server struct {
|
||||
cfg Config
|
||||
store DocumentStore
|
||||
keyGenerator KeyGenerator
|
||||
rateLimiter *RateLimiter
|
||||
staticDocs map[string]struct{}
|
||||
staticDir string
|
||||
nodeModules string
|
||||
}
|
||||
|
||||
func main() {
|
||||
seedMathRand()
|
||||
|
||||
configPath := "config.js"
|
||||
if len(os.Args) > 1 {
|
||||
configPath = os.Args[1]
|
||||
}
|
||||
|
||||
cfg, err := loadConfig(configPath)
|
||||
if err != nil {
|
||||
log.Fatalf("load config: %v", err)
|
||||
}
|
||||
|
||||
if cfg.Storage.Type == "" {
|
||||
cfg.Storage.Type = "redis"
|
||||
}
|
||||
if cfg.Storage.Type != "redis" {
|
||||
log.Fatalf("unsupported storage type %q: only redis is implemented in the Go backend", cfg.Storage.Type)
|
||||
}
|
||||
|
||||
store, err := NewRedisStore(cfg.Storage)
|
||||
if err != nil {
|
||||
log.Fatalf("connect redis: %v", err)
|
||||
}
|
||||
|
||||
keyGenerator, err := newKeyGenerator(cfg.KeyGenerator)
|
||||
if err != nil {
|
||||
log.Fatalf("configure key generator: %v", err)
|
||||
}
|
||||
|
||||
srv := &Server{
|
||||
cfg: cfg,
|
||||
store: store,
|
||||
keyGenerator: keyGenerator,
|
||||
rateLimiter: NewRateLimiter(cfg.RateLimits),
|
||||
staticDocs: make(map[string]struct{}, len(cfg.Documents)),
|
||||
staticDir: filepath.Join(".", "static"),
|
||||
nodeModules: filepath.Join(".", "node_modules"),
|
||||
}
|
||||
|
||||
if err := srv.loadStaticDocuments(); err != nil {
|
||||
log.Fatalf("load static documents: %v", err)
|
||||
}
|
||||
|
||||
address := net.JoinHostPort(cfg.Host, strconv.Itoa(cfg.Port))
|
||||
log.Printf("listening on %s", address)
|
||||
serve := &http.Server{
|
||||
Addr: address,
|
||||
Handler: srv.routes(),
|
||||
ReadHeaderTimeout: 5 * time.Second,
|
||||
}
|
||||
|
||||
if err := serve.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) {
|
||||
log.Fatalf("listen: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func seedMathRand() {
|
||||
var seedBytes [8]byte
|
||||
if _, err := rand.Read(seedBytes[:]); err == nil {
|
||||
var seed int64
|
||||
for _, b := range seedBytes {
|
||||
seed = (seed << 8) | int64(b)
|
||||
}
|
||||
mathrand.Seed(seed)
|
||||
return
|
||||
}
|
||||
mathrand.Seed(time.Now().UnixNano())
|
||||
}
|
||||
|
||||
func loadConfig(path string) (Config, error) {
|
||||
cfg := Config{
|
||||
Host: "localhost",
|
||||
Port: 7777,
|
||||
KeyLength: 10,
|
||||
MaxLength: 400000,
|
||||
StaticMaxAge: 86400,
|
||||
RecompressStaticAssets: true,
|
||||
KeyGenerator: KeyGeneratorConfig{Type: "random"},
|
||||
RateLimits: RateLimitsConfig{Categories: map[string]RateLimitBucket{}},
|
||||
Storage: StorageConfig{Type: "redis", Host: "localhost", Port: 6379},
|
||||
Documents: map[string]string{},
|
||||
}
|
||||
|
||||
if data, err := os.ReadFile(path); err == nil {
|
||||
if err := json.Unmarshal(data, &cfg); err != nil {
|
||||
return Config{}, err
|
||||
}
|
||||
} else if !errors.Is(err, os.ErrNotExist) {
|
||||
return Config{}, err
|
||||
}
|
||||
|
||||
applyEnvOverrides(&cfg)
|
||||
if cfg.Documents == nil {
|
||||
cfg.Documents = map[string]string{}
|
||||
}
|
||||
if cfg.RateLimits.Categories == nil {
|
||||
cfg.RateLimits.Categories = map[string]RateLimitBucket{}
|
||||
}
|
||||
if cfg.KeyGenerator.Type == "" {
|
||||
cfg.KeyGenerator.Type = "random"
|
||||
}
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
func applyEnvOverrides(cfg *Config) {
|
||||
if v := os.Getenv("HOST"); v != "" {
|
||||
cfg.Host = v
|
||||
}
|
||||
if v := os.Getenv("PORT"); v != "" {
|
||||
cfg.Port = atoiDefault(v, cfg.Port)
|
||||
}
|
||||
if v := os.Getenv("KEY_LENGTH"); v != "" {
|
||||
cfg.KeyLength = atoiDefault(v, cfg.KeyLength)
|
||||
}
|
||||
if v := os.Getenv("MAX_LENGTH"); v != "" {
|
||||
cfg.MaxLength = atoiDefault(v, cfg.MaxLength)
|
||||
}
|
||||
if v := os.Getenv("STATIC_MAX_AGE"); v != "" {
|
||||
cfg.StaticMaxAge = atoiDefault(v, cfg.StaticMaxAge)
|
||||
}
|
||||
if v := os.Getenv("RECOMPRESS_STATIC_ASSETS"); v != "" {
|
||||
cfg.RecompressStaticAssets = parseBool(v)
|
||||
}
|
||||
if v := os.Getenv("KEYGENERATOR_TYPE"); v != "" {
|
||||
cfg.KeyGenerator.Type = v
|
||||
}
|
||||
if v := os.Getenv("KEY_GENERATOR_KEYSPACE"); v != "" {
|
||||
cfg.KeyGenerator.Keyspace = v
|
||||
}
|
||||
if v := os.Getenv("STORAGE_TYPE"); v != "" {
|
||||
cfg.Storage.Type = v
|
||||
}
|
||||
if v := os.Getenv("STORAGE_HOST"); v != "" {
|
||||
cfg.Storage.Host = v
|
||||
}
|
||||
if v := os.Getenv("STORAGE_PORT"); v != "" {
|
||||
cfg.Storage.Port = atoiDefault(v, cfg.Storage.Port)
|
||||
}
|
||||
if v := os.Getenv("STORAGE_EXPIRE_SECONDS"); v != "" {
|
||||
cfg.Storage.Expire = atoiDefault(v, cfg.Storage.Expire)
|
||||
}
|
||||
if v := os.Getenv("STORAGE_DB"); v != "" {
|
||||
cfg.Storage.DB = atoiDefault(v, cfg.Storage.DB)
|
||||
}
|
||||
if v := os.Getenv("STORAGE_PASSWORD"); v != "" {
|
||||
cfg.Storage.Password = v
|
||||
}
|
||||
if v := os.Getenv("DOCUMENTS"); v != "" {
|
||||
cfg.Documents = parseDocuments(v)
|
||||
}
|
||||
applyRateLimitEnv(cfg)
|
||||
}
|
||||
|
||||
func applyRateLimitEnv(cfg *Config) {
|
||||
normal := cfg.RateLimits.Categories["normal"]
|
||||
if v := os.Getenv("RATE_LIMITS_NORMAL_TOTAL_REQUESTS"); v != "" {
|
||||
normal.TotalRequests = atoiDefault(v, normal.TotalRequests)
|
||||
}
|
||||
if v := os.Getenv("RATE_LIMITS_NORMAL_EVERY_MILLISECONDS"); v != "" {
|
||||
normal.Every = atoiDefault(v, normal.Every)
|
||||
}
|
||||
if normal.TotalRequests > 0 && normal.Every > 0 {
|
||||
cfg.RateLimits.Categories["normal"] = normal
|
||||
}
|
||||
if v := os.Getenv("RATE_LIMITS_WHITELIST"); v != "" {
|
||||
cfg.RateLimits.Whitelist = splitTrimmed(v)
|
||||
}
|
||||
if v := os.Getenv("RATE_LIMITS_BLACKLIST"); v != "" {
|
||||
cfg.RateLimits.Blacklist = splitTrimmed(v)
|
||||
}
|
||||
whitelist := cfg.RateLimits.Categories["whitelist"]
|
||||
if v := os.Getenv("RATE_LIMITS_WHITELIST_TOTAL_REQUESTS"); v != "" {
|
||||
whitelist.TotalRequests = atoiDefault(v, whitelist.TotalRequests)
|
||||
}
|
||||
if v := os.Getenv("RATE_LIMITS_WHITELIST_EVERY_MILLISECONDS"); v != "" {
|
||||
whitelist.Every = atoiDefault(v, whitelist.Every)
|
||||
}
|
||||
if whitelist.TotalRequests > 0 && whitelist.Every > 0 {
|
||||
cfg.RateLimits.Categories["whitelist"] = whitelist
|
||||
}
|
||||
blacklist := cfg.RateLimits.Categories["blacklist"]
|
||||
if v := os.Getenv("RATE_LIMITS_BLACKLIST_TOTAL_REQUESTS"); v != "" {
|
||||
blacklist.TotalRequests = atoiDefault(v, blacklist.TotalRequests)
|
||||
}
|
||||
if v := os.Getenv("RATE_LIMITS_BLACKLIST_EVERY_MILLISECONDS"); v != "" {
|
||||
blacklist.Every = atoiDefault(v, blacklist.Every)
|
||||
}
|
||||
if blacklist.TotalRequests > 0 && blacklist.Every > 0 {
|
||||
cfg.RateLimits.Categories["blacklist"] = blacklist
|
||||
}
|
||||
}
|
||||
|
||||
func newKeyGenerator(cfg KeyGeneratorConfig) (KeyGenerator, error) {
|
||||
switch cfg.Type {
|
||||
case "", "random":
|
||||
return RandomKeyGenerator{keyspace: cfg.Keyspace}, nil
|
||||
case "phonetic":
|
||||
return PhoneticKeyGenerator{}, nil
|
||||
case "dictionary":
|
||||
if cfg.Path == "" {
|
||||
return nil, errors.New("dictionary key generator requires path")
|
||||
}
|
||||
data, err := os.ReadFile(cfg.Path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
words := strings.Fields(string(data))
|
||||
if len(words) == 0 {
|
||||
return nil, errors.New("dictionary file is empty")
|
||||
}
|
||||
return DictionaryKeyGenerator{words: words}, nil
|
||||
default:
|
||||
return nil, fmt.Errorf("unsupported key generator %q", cfg.Type)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Server) loadStaticDocuments() error {
|
||||
ctx := context.Background()
|
||||
for name, path := range s.cfg.Documents {
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return fmt.Errorf("%s: %w", path, err)
|
||||
}
|
||||
if err := s.store.Set(ctx, name, string(data), true); err != nil {
|
||||
return fmt.Errorf("store %s: %w", name, err)
|
||||
}
|
||||
s.staticDocs[name] = struct{}{}
|
||||
log.Printf("loaded static document %s from %s", name, path)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Server) routes() http.Handler {
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/documents", s.handleDocumentsPost)
|
||||
mux.HandleFunc("/documents/", s.handleDocumentsGet)
|
||||
mux.HandleFunc("/raw/", s.handleRawGet)
|
||||
mux.Handle("/node_modules/", s.staticFileHandler(http.Dir(s.nodeModules), false))
|
||||
mux.Handle("/", s)
|
||||
if s.rateLimiter != nil {
|
||||
return s.rateLimiter.Middleware(mux)
|
||||
}
|
||||
return mux
|
||||
}
|
||||
|
||||
func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodGet && r.Method != http.MethodHead {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
|
||||
path := strings.TrimPrefix(r.URL.Path, "/")
|
||||
if path == "" {
|
||||
s.serveIndex(w, r)
|
||||
return
|
||||
}
|
||||
|
||||
if s.tryServeStatic(w, r, path) {
|
||||
return
|
||||
}
|
||||
|
||||
if !strings.Contains(path, "/") {
|
||||
s.serveIndex(w, r)
|
||||
return
|
||||
}
|
||||
|
||||
http.NotFound(w, r)
|
||||
}
|
||||
|
||||
func (s *Server) tryServeStatic(w http.ResponseWriter, r *http.Request, path string) bool {
|
||||
cleanPath := filepath.Clean(path)
|
||||
if strings.HasPrefix(cleanPath, "..") {
|
||||
http.NotFound(w, r)
|
||||
return true
|
||||
}
|
||||
localPath := filepath.Join(s.staticDir, cleanPath)
|
||||
if info, err := os.Stat(localPath); err == nil && !info.IsDir() {
|
||||
s.serveDiskFile(w, r, localPath)
|
||||
return true
|
||||
}
|
||||
|
||||
if file, err := embeddedStatic.Open(filepath.ToSlash(filepath.Join("static", cleanPath))); err == nil {
|
||||
_ = file.Close()
|
||||
s.serveEmbeddedFile(w, r, filepath.ToSlash(filepath.Join("static", cleanPath)))
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (s *Server) serveIndex(w http.ResponseWriter, r *http.Request) {
|
||||
indexPath := filepath.Join(s.staticDir, "index.html")
|
||||
if _, err := os.Stat(indexPath); err == nil {
|
||||
s.serveDiskFile(w, r, indexPath)
|
||||
return
|
||||
}
|
||||
s.serveEmbeddedFile(w, r, "static/index.html")
|
||||
}
|
||||
|
||||
func (s *Server) serveDiskFile(w http.ResponseWriter, r *http.Request, path string) {
|
||||
file, err := os.Open(path)
|
||||
if err != nil {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
info, err := file.Stat()
|
||||
if err != nil {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
|
||||
setStaticHeaders(w, path, s.cfg.StaticMaxAge)
|
||||
http.ServeContent(w, r, info.Name(), info.ModTime(), file)
|
||||
}
|
||||
|
||||
func (s *Server) serveEmbeddedFile(w http.ResponseWriter, r *http.Request, path string) {
|
||||
data, err := embeddedStatic.ReadFile(path)
|
||||
if err != nil {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
setStaticHeaders(w, path, s.cfg.StaticMaxAge)
|
||||
if ctype := mime.TypeByExtension(filepath.Ext(path)); ctype != "" {
|
||||
w.Header().Set("Content-Type", ctype)
|
||||
}
|
||||
http.ServeContent(w, r, filepath.Base(path), time.Time{}, strings.NewReader(string(data)))
|
||||
}
|
||||
|
||||
func setStaticHeaders(w http.ResponseWriter, path string, maxAge int) {
|
||||
if maxAge > 0 {
|
||||
w.Header().Set("Cache-Control", fmt.Sprintf("public, max-age=%d", maxAge))
|
||||
}
|
||||
if strings.HasSuffix(path, ".css") {
|
||||
w.Header().Set("Content-Type", "text/css; charset=utf-8")
|
||||
}
|
||||
if strings.HasSuffix(path, ".js") {
|
||||
w.Header().Set("Content-Type", "application/javascript; charset=utf-8")
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Server) staticFileHandler(fs http.FileSystem, stripPrefix bool) http.Handler {
|
||||
handler := http.FileServer(fs)
|
||||
if stripPrefix {
|
||||
return http.StripPrefix("/node_modules/", handler)
|
||||
}
|
||||
return http.StripPrefix("/node_modules/", handler)
|
||||
}
|
||||
|
||||
func (s *Server) handleDocumentsPost(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path != "/documents" {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
if r.Method != http.MethodPost {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
|
||||
body, err := s.readPostBody(r)
|
||||
if err != nil {
|
||||
writeJSON(w, http.StatusInternalServerError, map[string]string{"message": "Connection error."})
|
||||
return
|
||||
}
|
||||
if s.cfg.MaxLength > 0 && len(body) > s.cfg.MaxLength {
|
||||
writeJSON(w, http.StatusBadRequest, map[string]string{"message": "Document exceeds maximum length."})
|
||||
return
|
||||
}
|
||||
|
||||
key, err := s.chooseKey(r.Context())
|
||||
if err != nil {
|
||||
writeJSON(w, http.StatusInternalServerError, map[string]string{"message": "Error adding document."})
|
||||
return
|
||||
}
|
||||
if err := s.store.Set(r.Context(), key, body, false); err != nil {
|
||||
writeJSON(w, http.StatusInternalServerError, map[string]string{"message": "Error adding document."})
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, map[string]string{"key": key})
|
||||
}
|
||||
|
||||
func (s *Server) handleDocumentsGet(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodGet && r.Method != http.MethodHead {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
key := documentKey(strings.TrimPrefix(r.URL.Path, "/documents/"))
|
||||
skipExpire := s.isStaticDoc(key)
|
||||
val, ok, err := s.store.Get(r.Context(), key, skipExpire)
|
||||
if err != nil || !ok {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
if r.Method != http.MethodHead {
|
||||
_ = json.NewEncoder(w).Encode(map[string]string{"message": "Document not found."})
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
if r.Method == http.MethodHead {
|
||||
return
|
||||
}
|
||||
_ = json.NewEncoder(w).Encode(map[string]string{"data": val, "key": key})
|
||||
}
|
||||
|
||||
func (s *Server) handleRawGet(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodGet && r.Method != http.MethodHead {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
key := documentKey(strings.TrimPrefix(r.URL.Path, "/raw/"))
|
||||
skipExpire := s.isStaticDoc(key)
|
||||
val, ok, err := s.store.Get(r.Context(), key, skipExpire)
|
||||
if err != nil || !ok {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
if r.Method != http.MethodHead {
|
||||
_ = json.NewEncoder(w).Encode(map[string]string{"message": "Document not found."})
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "text/plain; charset=UTF-8")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
if r.Method != http.MethodHead {
|
||||
_, _ = io.WriteString(w, val)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Server) readPostBody(r *http.Request) (string, error) {
|
||||
contentType := r.Header.Get("Content-Type")
|
||||
mediaType, _, _ := mime.ParseMediaType(contentType)
|
||||
if mediaType == "multipart/form-data" {
|
||||
if err := r.ParseMultipartForm(int64(s.cfg.MaxLength) + 1024); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return r.FormValue("data"), nil
|
||||
}
|
||||
data, err := io.ReadAll(r.Body)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return string(data), nil
|
||||
}
|
||||
|
||||
func (s *Server) chooseKey(ctx context.Context) (string, error) {
|
||||
for i := 0; i < 64; i++ {
|
||||
key := s.keyGenerator.CreateKey(s.cfg.KeyLength)
|
||||
if _, exists, err := s.store.Get(ctx, key, true); err != nil {
|
||||
return "", err
|
||||
} else if !exists {
|
||||
return key, nil
|
||||
}
|
||||
}
|
||||
return "", errors.New("failed to allocate unique key")
|
||||
}
|
||||
|
||||
func (s *Server) isStaticDoc(key string) bool {
|
||||
_, ok := s.staticDocs[key]
|
||||
return ok
|
||||
}
|
||||
|
||||
func documentKey(path string) string {
|
||||
parts := strings.SplitN(path, ".", 2)
|
||||
return parts[0]
|
||||
}
|
||||
|
||||
func writeJSON(w http.ResponseWriter, status int, payload any) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(status)
|
||||
_ = json.NewEncoder(w).Encode(payload)
|
||||
}
|
||||
|
||||
type RateLimiter struct {
|
||||
cfg RateLimitsConfig
|
||||
mu sync.Mutex
|
||||
counters map[string]*rateCounter
|
||||
}
|
||||
|
||||
type rateCounter struct {
|
||||
count int
|
||||
expires time.Time
|
||||
}
|
||||
|
||||
func NewRateLimiter(cfg RateLimitsConfig) *RateLimiter {
|
||||
if len(cfg.Categories) == 0 {
|
||||
return nil
|
||||
}
|
||||
return &RateLimiter{
|
||||
cfg: cfg,
|
||||
counters: map[string]*rateCounter{},
|
||||
}
|
||||
}
|
||||
|
||||
func (r *RateLimiter) Middleware(next http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
|
||||
category, bucket, enabled := r.bucketFor(req)
|
||||
if enabled {
|
||||
ip := clientIP(req)
|
||||
if !r.allow(category, ip, bucket) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(http.StatusTooManyRequests)
|
||||
_ = json.NewEncoder(w).Encode(map[string]string{"message": "Rate limit exceeded."})
|
||||
return
|
||||
}
|
||||
}
|
||||
next.ServeHTTP(w, req)
|
||||
})
|
||||
}
|
||||
|
||||
func (r *RateLimiter) bucketFor(req *http.Request) (string, RateLimitBucket, bool) {
|
||||
ip := clientIP(req)
|
||||
if contains(r.cfg.Blacklist, ip) {
|
||||
if bucket, ok := r.cfg.Categories["blacklist"]; ok && bucket.TotalRequests > 0 && bucket.Every > 0 {
|
||||
return "blacklist", bucket, true
|
||||
}
|
||||
}
|
||||
if contains(r.cfg.Whitelist, ip) {
|
||||
if bucket, ok := r.cfg.Categories["whitelist"]; ok && bucket.TotalRequests > 0 && bucket.Every > 0 {
|
||||
return "whitelist", bucket, true
|
||||
}
|
||||
}
|
||||
bucket, ok := r.cfg.Categories["normal"]
|
||||
if !ok || bucket.TotalRequests <= 0 || bucket.Every <= 0 {
|
||||
return "", RateLimitBucket{}, false
|
||||
}
|
||||
return "normal", bucket, true
|
||||
}
|
||||
|
||||
func (r *RateLimiter) allow(category, ip string, bucket RateLimitBucket) bool {
|
||||
now := time.Now()
|
||||
key := category + ":" + ip
|
||||
window := time.Duration(bucket.Every) * time.Millisecond
|
||||
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
|
||||
counter, ok := r.counters[key]
|
||||
if !ok || now.After(counter.expires) {
|
||||
r.counters[key] = &rateCounter{count: 1, expires: now.Add(window)}
|
||||
return true
|
||||
}
|
||||
if counter.count >= bucket.TotalRequests {
|
||||
return false
|
||||
}
|
||||
counter.count++
|
||||
return true
|
||||
}
|
||||
|
||||
func clientIP(r *http.Request) string {
|
||||
host, _, err := net.SplitHostPort(r.RemoteAddr)
|
||||
if err == nil {
|
||||
return host
|
||||
}
|
||||
return r.RemoteAddr
|
||||
}
|
||||
|
||||
func parseBool(v string) bool {
|
||||
parsed, err := strconv.ParseBool(v)
|
||||
return err == nil && parsed
|
||||
}
|
||||
|
||||
func atoiDefault(v string, fallback int) int {
|
||||
parsed, err := strconv.Atoi(v)
|
||||
if err != nil {
|
||||
return fallback
|
||||
}
|
||||
return parsed
|
||||
}
|
||||
|
||||
func splitTrimmed(v string) []string {
|
||||
parts := strings.Split(v, ",")
|
||||
out := make([]string, 0, len(parts))
|
||||
for _, part := range parts {
|
||||
part = strings.TrimSpace(part)
|
||||
if part != "" {
|
||||
out = append(out, part)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func contains(values []string, target string) bool {
|
||||
for _, value := range values {
|
||||
if value == target {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func parseDocuments(v string) map[string]string {
|
||||
docs := map[string]string{}
|
||||
for _, entry := range strings.Split(v, ",") {
|
||||
entry = strings.TrimSpace(entry)
|
||||
if entry == "" {
|
||||
continue
|
||||
}
|
||||
parts := strings.SplitN(entry, "=", 2)
|
||||
if len(parts) != 2 {
|
||||
continue
|
||||
}
|
||||
key := strings.TrimSpace(parts[0])
|
||||
value := strings.TrimSpace(parts[1])
|
||||
if key != "" && value != "" {
|
||||
docs[key] = value
|
||||
}
|
||||
}
|
||||
return docs
|
||||
}
|
||||
3412
package-lock.json
generated
3412
package-lock.json
generated
File diff suppressed because it is too large
Load diff
21
package.json
21
package.json
|
|
@ -14,22 +14,9 @@
|
|||
},
|
||||
"main": "haste",
|
||||
"dependencies": {
|
||||
"busboy": "1.6.0",
|
||||
"connect": "^3.7.0",
|
||||
"connect-ratelimit": "0.0.7",
|
||||
"connect-route": "0.1.5",
|
||||
"jquery": "^4.0.0",
|
||||
"memcached": "^2.2.2",
|
||||
"pg": "^8.0.0",
|
||||
"redis": "5.11.0",
|
||||
"redis-url": "1.2.1",
|
||||
"st": "^3.0.0",
|
||||
"uglify-js": "3.19.3",
|
||||
"winston": "^3.0.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"mocha": "^11.0.0"
|
||||
"jquery": "^4.0.0"
|
||||
},
|
||||
"devDependencies": {},
|
||||
"bundledDependencies": [],
|
||||
"bin": {
|
||||
"haste-server": "./server.js"
|
||||
|
|
@ -43,8 +30,8 @@
|
|||
"lib": "./lib"
|
||||
},
|
||||
"scripts": {
|
||||
"start": "node server.js",
|
||||
"test": "mocha --recursive"
|
||||
"start": "./haste-server",
|
||||
"test": "go test ./..."
|
||||
},
|
||||
"bundleDependencies": []
|
||||
}
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@
|
|||
|
||||
<script type="text/javascript" src="node_modules/jquery/dist/jquery.min.js"></script>
|
||||
<script type="text/javascript" src="highlight.min.js"></script>
|
||||
<script type="text/javascript" src="application.min.js"></script>
|
||||
<script type="text/javascript" src="application.js"></script>
|
||||
|
||||
<meta name="robots" content="noindex,nofollow"/>
|
||||
|
||||
|
|
|
|||
|
|
@ -1,26 +0,0 @@
|
|||
/* global describe, it */
|
||||
|
||||
var assert = require('assert');
|
||||
|
||||
var DocumentHandler = require('../lib/document_handler');
|
||||
var Generator = require('../lib/key_generators/random');
|
||||
|
||||
describe('document_handler', function() {
|
||||
|
||||
describe('randomKey', function() {
|
||||
|
||||
it('should choose a key of the proper length', function() {
|
||||
var gen = new Generator();
|
||||
var dh = new DocumentHandler({ keyLength: 6, keyGenerator: gen });
|
||||
assert.equal(6, dh.acceptableKey().length);
|
||||
});
|
||||
|
||||
it('should choose a default key length', function() {
|
||||
var gen = new Generator();
|
||||
var dh = new DocumentHandler({ keyGenerator: gen });
|
||||
assert.equal(dh.keyLength, DocumentHandler.defaultKeyLength);
|
||||
});
|
||||
|
||||
});
|
||||
|
||||
});
|
||||
|
|
@ -1,34 +0,0 @@
|
|||
/* global describe, it */
|
||||
|
||||
const assert = require('assert');
|
||||
|
||||
const fs = require('fs');
|
||||
|
||||
const Generator = require('../../lib/key_generators/dictionary');
|
||||
|
||||
describe('DictionaryGenerator', function() {
|
||||
describe('options', function() {
|
||||
it('should throw an error if given no options', () => {
|
||||
assert.throws(() => {
|
||||
new Generator();
|
||||
}, Error);
|
||||
});
|
||||
|
||||
it('should throw an error if given no path', () => {
|
||||
assert.throws(() => {
|
||||
new Generator({});
|
||||
}, Error);
|
||||
});
|
||||
});
|
||||
describe('generation', function() {
|
||||
it('should return a key of the proper number of words from the given dictionary', () => {
|
||||
const path = '/tmp/haste-server-test-dictionary';
|
||||
const words = ['cat'];
|
||||
fs.writeFileSync(path, words.join('\n'));
|
||||
|
||||
const gen = new Generator({path}, () => {
|
||||
assert.equal('catcatcat', gen.createKey(3));
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
@ -1,35 +0,0 @@
|
|||
/* global describe, it */
|
||||
|
||||
const assert = require('assert');
|
||||
|
||||
const Generator = require('../../lib/key_generators/phonetic');
|
||||
|
||||
const vowels = 'aeiou';
|
||||
const consonants = 'bcdfghjklmnpqrstvwxyz';
|
||||
|
||||
describe('PhoneticKeyGenerator', () => {
|
||||
describe('generation', () => {
|
||||
it('should return a key of the proper length', () => {
|
||||
const gen = new Generator();
|
||||
assert.equal(6, gen.createKey(6).length);
|
||||
});
|
||||
|
||||
it('should alternate consonants and vowels', () => {
|
||||
const gen = new Generator();
|
||||
|
||||
const key = gen.createKey(3);
|
||||
|
||||
// if it starts with a consonant, we expect cvc
|
||||
// if it starts with a vowel, we expect vcv
|
||||
if(consonants.includes(key[0])) {
|
||||
assert.ok(consonants.includes(key[0]));
|
||||
assert.ok(consonants.includes(key[2]));
|
||||
assert.ok(vowels.includes(key[1]));
|
||||
} else {
|
||||
assert.ok(vowels.includes(key[0]));
|
||||
assert.ok(vowels.includes(key[2]));
|
||||
assert.ok(consonants.includes(key[1]));
|
||||
}
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
@ -1,24 +0,0 @@
|
|||
/* global describe, it */
|
||||
|
||||
const assert = require('assert');
|
||||
|
||||
const Generator = require('../../lib/key_generators/random');
|
||||
|
||||
describe('RandomKeyGenerator', () => {
|
||||
describe('generation', () => {
|
||||
it('should return a key of the proper length', () => {
|
||||
const gen = new Generator();
|
||||
assert.equal(gen.createKey(6).length, 6);
|
||||
});
|
||||
|
||||
it('should use a key from the given keyset if given', () => {
|
||||
const gen = new Generator({keyspace: 'A'});
|
||||
assert.equal(gen.createKey(6), 'AAAAAA');
|
||||
});
|
||||
|
||||
it('should not use a key from the given keyset if not given', () => {
|
||||
const gen = new Generator({keyspace: 'A'});
|
||||
assert.ok(!gen.createKey(6).includes('B'));
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
@ -1,54 +0,0 @@
|
|||
/* global it, describe, afterEach */
|
||||
|
||||
var assert = require('assert');
|
||||
|
||||
var winston = require('winston');
|
||||
winston.remove(winston.transports.Console);
|
||||
|
||||
var RedisDocumentStore = require('../lib/document_stores/redis');
|
||||
|
||||
describe('redis_document_store', function() {
|
||||
|
||||
/* reconnect to redis on each test */
|
||||
afterEach(function() {
|
||||
if (RedisDocumentStore.client) {
|
||||
RedisDocumentStore.client.quit();
|
||||
RedisDocumentStore.client = false;
|
||||
}
|
||||
});
|
||||
|
||||
describe('set', function() {
|
||||
|
||||
it('should be able to set a key and have an expiration set', function(done) {
|
||||
var store = new RedisDocumentStore({ expire: 10 });
|
||||
store.set('hello1', 'world', function() {
|
||||
RedisDocumentStore.client.ttl('hello1', function(err, res) {
|
||||
assert.ok(res > 1);
|
||||
done();
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
it('should not set an expiration when told not to', function(done) {
|
||||
var store = new RedisDocumentStore({ expire: 10 });
|
||||
store.set('hello2', 'world', function() {
|
||||
RedisDocumentStore.client.ttl('hello2', function(err, res) {
|
||||
assert.equal(-1, res);
|
||||
done();
|
||||
});
|
||||
}, true);
|
||||
});
|
||||
|
||||
it('should not set an expiration when expiration is off', function(done) {
|
||||
var store = new RedisDocumentStore({ expire: false });
|
||||
store.set('hello3', 'world', function() {
|
||||
RedisDocumentStore.client.ttl('hello3', function(err, res) {
|
||||
assert.equal(-1, res);
|
||||
done();
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
});
|
||||
|
||||
});
|
||||
Loading…
Add table
Add a link
Reference in a new issue