Skip to content
Snippets Groups Projects

Compare revisions

Changes are shown as if the source revision was being merged into the target revision. Learn more about comparing revisions.

Source

Select target project
No results found

Target

Select target project
  • whats-this/cdn-origin
  • easrng/cdn-origin
2 results
Show changes
Commits on Source (28)
Expand all Hide whitespace changes
Inline Side-by-side
Showing
with 886 additions and 460 deletions
.gitignore
View file @ 3d53bc53
......@@ -15,12 +15,15 @@
# Built package binary
cdn-origin
main
# Package configuration file (not including sample configuration)
cdn-origin.toml
config.toml
config.prod.toml
# MaxMind GeoLite2 Country database files
GeoLite2-Country.mmdb
# JetBrains .idea
.idea/
Dockerfile
View file @ 3d53bc53
FROM golang:alpine
FROM golang:1.21-alpine
COPY cdn-origin.go $GOPATH/src/github.com/whats-this/cdn-origin/
COPY prometheus/ $GOPATH/src/github.com/whats-this/cdn-origin/prometheus/
COPY weed/ $GOPATH/src/github.com/whats-this/cdn-origin/weed/
RUN apk add --no-cache --virtual .build-deps git build-base
RUN apk add --no-cache --virtual .build-deps git && \
COPY go.mod /git/owo.codes/whats-this/cdn-origin/
COPY go.sum /git/owo.codes/whats-this/cdn-origin/
COPY main.go /git/owo.codes/whats-this/cdn-origin/
COPY lib /git/owo.codes/whats-this/cdn-origin/lib
go-wrapper download github.com/lib/pq && \
go-wrapper download github.com/prometheus/client_golang/prometheus && \
go-wrapper download github.com/Sirupsen/logrus && \
go-wrapper download github.com/spf13/pflag && \
go-wrapper download github.com/spf13/viper && \
go-wrapper download github.com/valyala/fasthttp && \
go-wrapper install github.com/whats-this/cdn-origin && \
RUN cd /git/owo.codes/whats-this/cdn-origin && \
go mod download && \
go build main.go && \
apk del .build-deps
WORKDIR $GOPATH/src/github.com/whats-this/cdn-origin
ENTRYPOINT ["go-wrapper", "run"]
WORKDIR /git/owo.codes/whats-this/cdn-origin
ENTRYPOINT ["./main"]
LICENSE
View file @ 3d53bc53
The MIT License (MIT)
Copyright (c) 2017 Dean Sheather, whats-this
Copyright (c) 2019 Dean Sheather, whats-this
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
......
README.md
View file @ 3d53bc53
# Whats-This CDN Origin
# cdn-origin
Simple but quick Golang webserver that serves requests to get files and
redirects from a [PostgreSQL](https://www.postgresql.org) and
[SeaweedFS](https://github.com/chrislusf/seaweedfs) backend.
redirects from [PostgreSQL](https://www.postgresql.org).
### Features
- Serves files, short URLs and "tombstones" (deleted file markers)
- Allows for URL previewing on short URLs (add `?preview`)
- Allows for thumbnail generation on images via external thumbnailer service (if
enabled, add `?thumbnail`)
- Can be configured to store generalized metrics
### Requirements
- PostgreSQL server with `objects` table
- SeaweedFS cluster with files in the `objects` table
- Access to the folder where the files are stored
- If metrics support is desired, you will need an Elasticsearch server setup as
noted below
- If thumbnail support is desired, you will need a webserver with an endpoint
that returns a thumbnail from raw POSTed image data (`jpeg`, `gif`, `png`,
`webp`) such as
[thumbnail-service](https://owo.codes/whats-this/thumbnail-service). We
recommend that the service outputs 200x200 (max) JPEG images
### Usage
```
$ go get -u github.com/whats-this/cdn-origin
# With configuration file
$ cp $GOPATH/src/github.com/whats-this/cdn-origin/cdn-origin.sample.toml /etc/cdn-origin/cdn-origin.toml
$ vim /etc/cdn-origin/cdn-origin.toml
$ cdn-origin
# With environment variables
$ set DATABASE_CONNECTION_URL="postgres://postgres@localhost/data?sslmode=disable"
$ set SEAWEED_MASTER_URL="http://localhost:9333"
$ ...
$ cdn-origin
# With flags
$ cdn-origin \
--database-connection-url="postgres://postgres@localhost/data?sslmode=disable" \
--seaweed-master-url="http://localhost:9333" \
...
# Flags take precedence over environment variables, which take precedence over config files
$ git clone https://owo.codes/whats-this/cdn-origin.git
$ cd cdn-origin
$ cp config.sample.toml config.toml
$ vim config.toml
$ go build main.go
$ ./main --config-file "./config.toml"
```
Information about configuration variables and their purpose can be found in
[cdn-origin.sample.toml](cdn-origin.sample.toml). Configuration is handled by
[Viper](https://github.com/spf13/viper).
### Metrics
If `metrics.enable` is `true`, request metadata will be indexed in the provided
......@@ -55,15 +50,12 @@ Elaticsearch server in the following format:
The index and `@timestamp` pipeline are created automatically if `cdn-origin`
has permission. Alternatively, the mapping and pipeline can be created by other
means using the `.json` files in [metrics/](metrics).
means using the `.json` files in [lib/metrics/](lib/metrics).
### TODO
- [ ] Process chunked files stored in SeaweedFS, similar to [how SeaweedFS cli
handles it](https://github.com/chrislusf/seaweedfs/wiki/Large-File-Handling)
- [ ] Add TTL to volume cache
- [ ] Write tests
- [ ] Add thumbnail functionality (SeaweedFS supports this)
- `OPTIONS`/`HEAD` support
- Write tests
### License
......
cdn-origin.go deleted 100644 → 0
View file @ b34068a9
package main
import (
"database/sql"
"fmt"
"html/template"
"net"
"strings"
"time"
"github.com/whats-this/cdn-origin/metrics"
"github.com/whats-this/cdn-origin/weed"
log "github.com/Sirupsen/logrus"
_ "github.com/lib/pq"
"github.com/spf13/pflag"
"github.com/spf13/viper"
"github.com/valyala/fasthttp"
)
// version is the current version of cdn-origin.
const version = "0.4.0"
// Default SeaweedFS fetch query parameters.
const defaultSeaweedFSQueryParameters = ""
// HTTP header strings.
const (
accept = "accept"
host = "host"
location = "Location"
)
// redirectHTML is the html/template template for generating redirect HTML.
const redirectHTML = `<html><head><meta charset="UTF-8" /><meta http-equiv=refresh content="0; url={{.}}" /><script type="text/javascript">window.location.href="{{.}}"</script><title>Redirect</title></head><body><p>If you are not redirected automatically, click <a href="{{.}}">here</a> to go to the destination.</p></body></html>`
var redirectHTMLTemplate *template.Template
// redirectPreviewHTML is the html/template template for generating redirect preview HTML.
const redirectPreviewHTML = `<html><head><meta charset="UTF-8" /><title>Redirect Preview</title></head><body><p>This link goes to <code>{{.}}</code>. If you would like to visit this link, click <a href="{{.}}">here</a> to go to the destination.</p></body></html>`
var redirectPreviewHTMLTemplate *template.Template
func init() {
// Read in configuration
flags := pflag.NewFlagSet("cdn-origin", pflag.ExitOnError)
// database.connectionURL* (string): PostgreSQL connection URL
flags.String("database-connection-url", "", "* PostgreSQL connection URL")
viper.BindPFlag("database.connectionURL", flags.Lookup("database-connection-url"))
viper.BindEnv("database.connectionURL", "DATABASE_CONNECTION_URL")
// debug (bool=false): enable debug mode (logs requests and prints other information)
flags.Bool("debug", false, "Enable debug mode (logs requests and prints other information)")
viper.BindPFlag("debug", flags.Lookup("debug"))
viper.BindEnv("debug", "DEBUG")
// http.compressResponse (bool=false): enable transparent response compression
flags.Bool("compress-response", false, "Enable transparent response compression")
viper.BindPFlag("http.compressResponse", flags.Lookup("compress-response"))
viper.BindEnv("http.compressResponse", "HTTP_COMPRESS_RESPONSE")
// http.listenAddress (string=":8080"): TCP address to listen to for HTTP requests
flags.String("listen-address", ":8080", "TCP address to listen to for HTTP requests")
viper.BindPFlag("http.listenAddress", flags.Lookup("listen-address"))
viper.BindEnv("http.listenAddress", "HTTP_LISTEN_ADDRESS")
// http.trustProxy (bool=false): trust X-Forwarded-For header from proxy
flags.Bool("trust-proxy", false, "Trust X-Forwarded-For header from proxy")
viper.BindPFlag("http.trustProxy", flags.Lookup("trust-proxy"))
viper.BindEnv("http.trustProxy", "HTTP_TRUST_PROXY")
// metrics.enable (bool=false): enable anonymized request recording (country code, hostname, object type, status
// code)
flags.Bool("enable-metrics", false,
"Enable anonymized request recording (status code, country code, object type, hostname)")
viper.BindPFlag("metrics.enable", flags.Lookup("enable-metrics"))
viper.BindEnv("metrics.enable", "ENABLE_METRICS")
// metrics.elasticURL (string): Elasticsearch URL to connect to for metrics, required if metrics.enable == true
flags.String("elastic-url", "", "Elastic URL to connect to for metrics")
viper.BindPFlag("metrics.elasticURL", flags.Lookup("elastic-url"))
viper.BindEnv("metrics.elasticURL", "ELASTIC_URL")
// metrics.enableHostnameWhitelist (bool=false): enable hostname whitelist for metrics
// (metrics.hostnameWhitelist)
flags.Bool("metrics-enable-whitelist", false, "Enable hostname whitelist for metrics")
viper.BindPFlag("metrics.enableHostnameWhitelist", flags.Lookup("metrics-enable-whitelist"))
viper.BindEnv("metrics.enableHostnameWhitelist", "METRICS_ENABLE_WHITELIST")
// metrics.hostnameWhitelist (string[]=[]): hostnames to whitelist for metrics (config file only)
// metrics.maxmindDBLocation (string): location of MaxMind GeoLite2 Country database file (.mmdb) on disk, if
// empty country codes will not be recorded by metrics collector
flags.String("maxmind-db-location", "", "Location of MaxMind GeoLite2 Country database on disk")
viper.BindPFlag("metrics.maxmindDBLocation", flags.Lookup("maxmind-db-location"))
viper.BindEnv("metrics.maxmindDBLocation", "MAXMIND_DB_LOCATION")
// seaweed.masterURL* (string): SeaweedFS master URL
flags.String("seaweed-master-url", "", "* SeaweedFS master URL")
viper.BindPFlag("seaweed.masterURL", flags.Lookup("seaweed-master-url"))
viper.BindEnv("seaweed.masterURL", "SEAWEED_MASTER_URL")
// Configuration file settings
viper.SetConfigType("toml")
viper.SetConfigName("cdn-origin")
viper.AddConfigPath(".")
viper.AddConfigPath("/etc/cdn-origin/")
if err := viper.ReadInConfig(); err != nil {
if _, ok := err.(viper.ConfigFileNotFoundError); !ok {
log.WithError(err).Fatal("failed to read in configuration")
}
}
// Enable debug mode
if viper.GetBool("debug") {
log.SetLevel(log.DebugLevel)
}
// Print configuration variables to debug
log.WithFields(log.Fields{
"database.connectionURL": viper.GetString("database.connectionURL"),
"debug": viper.GetBool("debug"),
"http.compressResponse": viper.GetBool("http.compressResponse"),
"http.listenAddress": viper.GetString("http.listenAddress"),
"metrics.enable": viper.GetBool("metrics.enable"),
"metrics.elasticURL": viper.GetString("metrics.elasticURL"),
"len(metrics.hostnameWhitelist)": len(viper.GetStringSlice("metrics.hostnameWhitelist")),
"metrics.enableHostnameWhitelist": viper.GetString("metrics.enableHostnameWhitelist"),
"seaweed.masterURL": viper.GetString("seaweed.masterURL"),
}).Debug("retrieved configuration")
// Ensure required configuration variables are set
if viper.GetString("database.connectionURL") == "" {
log.Fatal("database.connectionURL is required")
}
if viper.GetString("seaweed.masterURL") == "" {
log.Fatal("seaweed.masterURL is required")
}
if viper.GetBool("metrics.enable") && viper.GetString("metrics.elasticURL") == "" {
log.Fatal("metrics.elasticURL is required when metrics are enabled")
}
// Parse redirect templates
var err error
redirectHTMLTemplate, err = template.New("redirectHTML").Parse(redirectHTML)
if err != nil {
log.WithError(err).Fatal("failed to parse redirectHTML template")
}
redirectPreviewHTMLTemplate, err = template.New("redirectPreviewHTML").Parse(redirectPreviewHTML)
if err != nil {
log.WithError(err).Fatal("failed to parse redirectPreviewHTML template")
}
}
var db *sql.DB
var collector *metrics.Collector
var seaweed *weed.Seaweed
func main() {
var err error
// Attempt to connect to SeaweedFS master
seaweed = weed.New(viper.GetString("seaweed.masterURL"), time.Second*5)
err = seaweed.Ping()
if err != nil {
log.WithError(err).Fatal("failed to ping SeaweedFS master")
}
// Connect to PostgreSQL database
// TODO: abstractify database connection
// TODO: create table if it doesn't exist
db, err = sql.Open("postgres", viper.GetString("database.connectionURL"))
if err != nil {
log.WithError(err).Fatal("failed to open database connection")
}
// Setup metrics collector
if viper.GetBool("metrics.enable") {
hostnameWhitelist := []string{}
if viper.GetBool("metrics.enableHostnameWhitelist") {
switch w := viper.Get("metrics.hostnameWhitelist").(type) {
case []interface{}:
for _, s := range w {
hostnameWhitelist = append(hostnameWhitelist, strings.TrimSpace(fmt.Sprint(s)))
}
break
default:
log.Fatal("metrics.hostnameWhitelist is not an array")
}
}
collector, err = metrics.New(
viper.GetString("metrics.elasticURL"),
viper.GetString("metrics.maxmindDBLocation"),
viper.GetBool("metrics.enableHostnameWhitelist"),
hostnameWhitelist,
)
if err != nil {
log.WithError(err).Fatal("failed to setup metrics collector")
}
}
// Launch server
h := requestHandler
if viper.GetBool("http.compressResponse") {
h = fasthttp.CompressHandler(h)
}
log.Info("Attempting to listen on " + viper.GetString("http.listenAddress"))
server := &fasthttp.Server{
Handler: h,
Name: "whats-this/cdn-origin v" + version,
ReadBufferSize: 1024 * 6, // 6 KB
ReadTimeout: time.Minute * 30,
WriteTimeout: time.Minute * 30,
GetOnly: true, // TODO: OPTIONS/HEAD requests
LogAllErrors: log.GetLevel() == log.DebugLevel,
DisableHeaderNamesNormalizing: false,
Logger: log.New(),
}
if err := server.ListenAndServe(viper.GetString("http.listenAddress")); err != nil {
log.WithError(err).Fatal("error in server.ListenAndServe")
}
}
func recordMetrics(ctx *fasthttp.RequestCtx, objectType string) {
if !viper.GetBool("metrics.enable") {
return
}
var remoteIP net.IP
if viper.GetBool("http.trustProxy") {
ipString := string(ctx.Request.Header.Peek("X-Forwarded-For"))
remoteIP = net.ParseIP(strings.Split(ipString, ",")[0])
} else {
remoteIP = ctx.RemoteIP()
}
remoteIP = net.ParseIP("51.15.83.140")
hostBytes := ctx.Request.Header.Peek(host)
if len(hostBytes) != 0 {
go func() {
// Check hostname
hostStr, isValid := collector.MatchHostname(string(hostBytes))
if !isValid {
return
}
// Get country code of visitor
countryCode, err := collector.GetCountryCode(remoteIP)
if err != nil {
// Don't log the error here, it might contain an IP address
log.Warn("failed to get country code for IP, omitting from record")
}
record := metrics.GetRecord()
record.CountryCode = countryCode
record.Hostname = hostStr
record.ObjectType = objectType
record.StatusCode = ctx.Response.StatusCode()
err = collector.Put(record)
if err != nil {
log.WithError(err).Warn("failed to collect record")
}
log.Debug("successfully collected metrics")
}()
}
}
func requestHandler(ctx *fasthttp.RequestCtx) {
metricsObjectType := ""
// Log requests in debug mode, wrapped in an if statement to prevent unnecessary memory allocations
if log.GetLevel() == log.DebugLevel {
log.WithFields(log.Fields{
"connRequestNumber": ctx.ConnRequestNum(),
"method": string(ctx.Method()),
"queryString": ctx.QueryArgs(),
"remoteIP": ctx.RemoteIP(),
"requestURI": string(ctx.RequestURI()),
}).Debug("request received")
}
// Fetch object from database
var backendFileID sql.NullString
var contentType sql.NullString
var destURL sql.NullString
var objectType int
err := db.QueryRow(
`SELECT backend_file_id, content_type, dest_url, "type" FROM objects WHERE bucket_key=$1 LIMIT 1`,
fmt.Sprintf("public%s", ctx.Path()),
).Scan(&backendFileID, &contentType, &destURL, &objectType)
switch {
case err == sql.ErrNoRows:
ctx.SetStatusCode(fasthttp.StatusNotFound)
ctx.SetContentType("text/plain; charset=utf8")
fmt.Fprintf(ctx, "404 Not Found: %s", ctx.Path())
recordMetrics(ctx, metricsObjectType)
return
case err != nil:
log.WithError(err).Error("failed to run SELECT query on database")
ctx.SetStatusCode(fasthttp.StatusInternalServerError)
ctx.SetContentType("text/plain; charset=utf8")
fmt.Fprint(ctx, "500 Internal Server Error")
ctx.SetUserValue("object_type", "file")
recordMetrics(ctx, metricsObjectType)
return
}
switch objectType {
case 0: // file
metricsObjectType = "file"
// Get object from SeaweedFS and write to response
if !backendFileID.Valid {
log.WithField("bucket_key", string(ctx.Path())).Warn("found file object with NULL backend_file_id")
ctx.SetStatusCode(fasthttp.StatusInternalServerError)
ctx.SetContentType("text/plain; charset=utf8")
fmt.Fprint(ctx, "500 Internal Server Error")
recordMetrics(ctx, metricsObjectType)
return
}
// TODO: ?thumbnail query parameter for images
statusCode, contentSize, err := seaweed.Get(ctx, backendFileID.String, defaultSeaweedFSQueryParameters)
if err != nil {
log.WithError(err).Warn("failed to retrieve file from SeaweedFS volume server")
ctx.SetStatusCode(fasthttp.StatusInternalServerError)
ctx.SetContentType("text/plain; charset=utf8")
fmt.Fprint(ctx, "500 Internal Server Error")
recordMetrics(ctx, metricsObjectType)
return
}
if statusCode != fasthttp.StatusOK {
log.WithFields(log.Fields{
"expected": fasthttp.StatusOK,
"got": statusCode,
}).Warn("unexpected status code while retrieving file from SeaweedFS volume server")
ctx.SetStatusCode(fasthttp.StatusInternalServerError)
ctx.SetContentType("text/plain; charset=utf8")
fmt.Fprint(ctx, "500 Internal Server Error")
recordMetrics(ctx, metricsObjectType)
return
}
if contentType.Valid {
ctx.SetContentType(contentType.String)
} else {
ctx.SetContentType("application/octet-stream")
}
ctx.Response.Header.SetContentLength(contentSize)
recordMetrics(ctx, metricsObjectType)
case 1: // redirect
metricsObjectType = "redirect"
if !destURL.Valid {
log.Warn("encountered redirect object with NULL dest_url")
ctx.SetStatusCode(fasthttp.StatusInternalServerError)
ctx.SetContentType("text/plain; charset=utf8")
fmt.Fprint(ctx, "500 Internal Server Error")
recordMetrics(ctx, metricsObjectType)
return
}
previewMode := ctx.QueryArgs().Has("preview")
var err error
if previewMode {
err = redirectPreviewHTMLTemplate.Execute(ctx, destURL.String)
} else {
err = redirectHTMLTemplate.Execute(ctx, destURL.String)
}
if err != nil {
log.WithFields(log.Fields{
"error": err,
"dest_url": destURL.String,
"preview": ctx.QueryArgs().Has("preview"),
}).Warn("failed to generate HTML redirect page to send to client")
ctx.SetContentType("text/plain; charset=utf8")
fmt.Fprintf(ctx, "Failed to generate HTML redirect page, destination URL: %s", destURL.String)
recordMetrics(ctx, metricsObjectType)
return
}
ctx.SetContentType("text/html; charset=ut8")
if !previewMode {
ctx.SetStatusCode(fasthttp.StatusFound)
ctx.Response.Header.Set(location, destURL.String)
}
recordMetrics(ctx, metricsObjectType)
}
}
cdn-origin.sample.toml → config.sample.toml
View file @ 3d53bc53
# Enable debug mode (logs requests and prints other information)
debug = false
[log]
# Log level (5=panic, 4=fatal, 3=error, 2=warn, 1=info, 0=debug)
level = 1
[database]
# PostgreSQL connection URL, see
......@@ -7,6 +8,9 @@ debug = false
# more information
connectionURL = "postgres://postgres:password@localhost/whats_this?sslmode=disable"
# Bucket to serve objects from
objectBucket = "public"
[http]
# Enable transparent response compression (only when the client Accepts it)
compressResponse = false
......@@ -49,7 +53,21 @@ debug = false
# code data will be collected. https://dev.maxmind.com/geoip/geoip2/geolite2
maxmindDBLocation = "/var/data/maxmind/GeoLite2-Country.mmdb"
[seaweed]
# SeaweedFS master URL (used for looking up volumes, volumes must be
# accessible by their publicUrl)
masterURL = "http://localhost:9333"
[files]
# Storage location of the bucket on disk
storageLocation = "/var/data/buckets/public"
[thumbnails]
# Enable thumbnails? (add ?thumbnail to the end of a file object URL)
enable = true
# Thumbnailer URL. This is a endpoint that accepts a POST request containing
# raw image data and returns a thumbnail. For example, see
# https://owo.codes/whats-this/thumbnail-service.
thumbnailerURL = "http://localhost:8081/thumbnail"
# Enable thumbnail cache?
cacheEnable = true
# Thumbnail cache location (if enabled).
cacheLocation = "/tmp/thumbs"
go.mod 0 → 100644
View file @ 3d53bc53
module owo.codes/whats-this/cdn-origin
go 1.21
require (
github.com/lib/pq v1.10.9
github.com/oschwald/maxminddb-golang v1.12.0
github.com/pkg/errors v0.9.1
github.com/rs/zerolog v1.31.0
github.com/spf13/pflag v1.0.5
github.com/spf13/viper v1.17.0
github.com/valyala/fasthttp v1.50.0
gopkg.in/olivere/elastic.v5 v5.0.86
)
require (
github.com/andybalholm/brotli v1.0.6 // indirect
github.com/fsnotify/fsnotify v1.7.0 // indirect
github.com/hashicorp/hcl v1.0.0 // indirect
github.com/josharian/intern v1.0.0 // indirect
github.com/klauspost/compress v1.17.2 // indirect
github.com/magiconair/properties v1.8.7 // indirect
github.com/mailru/easyjson v0.7.7 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/mitchellh/mapstructure v1.5.0 // indirect
github.com/pelletier/go-toml/v2 v2.1.0 // indirect
github.com/sagikazarmark/locafero v0.3.0 // indirect
github.com/sagikazarmark/slog-shim v0.1.0 // indirect
github.com/sourcegraph/conc v0.3.0 // indirect
github.com/spf13/afero v1.10.0 // indirect
github.com/spf13/cast v1.5.1 // indirect
github.com/spf13/jwalterweatherman v1.1.0 // indirect
github.com/subosito/gotenv v1.6.0 // indirect
github.com/valyala/bytebufferpool v1.0.0 // indirect
go.uber.org/atomic v1.11.0 // indirect
go.uber.org/multierr v1.11.0 // indirect
golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa // indirect
golang.org/x/sys v0.14.0 // indirect
golang.org/x/text v0.14.0 // indirect
gopkg.in/ini.v1 v1.67.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)
go.sum 0 → 100644
View file @ 3d53bc53
This diff is collapsed.
lib/db/connect.go 0 → 100644
View file @ 3d53bc53
package db
import (
"database/sql"
// postgres driver for database/sql
_ "github.com/lib/pq"
)
// DB is the current database connection.
var DB *sql.DB
// Connect to the database using the given driver and connection URL.
func Connect(driver string, connectionURL string) error {
var err error
DB, err = sql.Open(driver, connectionURL)
if err != nil {
return err
}
return DB.Ping()
}
lib/db/models.go 0 → 100644
View file @ 3d53bc53
package db
import (
"time"
)
// Object represents a partial object from the database.
type Object struct {
ContentType *string `json:"content_type"`
DestURL *string `json:"dest_url"`
ObjectType int `json:"object_type"`
DeletedAt *time.Time `json:"deleted_at"`
DeleteReason *string `json:"delete_reason"`
MD5HashBytes []byte `json:"-"`
SHA256HashBytes []byte `json:"-"`
// Computed fields
MD5Hash *string `json:"md5_hash"`
SHA256Hash *string `json:"sha256_hash"`
}
lib/db/queries.go 0 → 100644
View file @ 3d53bc53
package db
import (
"database/sql"
"encoding/hex"
"fmt"
"github.com/lib/pq"
)
// SelectObjectByBucketKey returns an object from a bucket and a key.
func SelectObjectByBucketKey(bucket, key string) (Object, error) {
var object Object
var contentType sql.NullString
var destURL sql.NullString
var objectType int
var deletedAt pq.NullTime
var deleteReason sql.NullString
var md5Hash []byte
var sha256Hash []byte
err := DB.QueryRow(selectObjectByBucketKey, fmt.Sprintf("%s/%s", bucket, key)).
Scan(&contentType, &destURL, &objectType, &deletedAt, &deleteReason, &md5Hash, &sha256Hash)
if err != nil {
return object, err
}
// Populate object values
if contentType.Valid {
object.ContentType = &contentType.String
}
if destURL.Valid {
object.DestURL = &destURL.String
}
if deletedAt.Valid {
object.DeletedAt = &deletedAt.Time
if deleteReason.Valid {
object.DeleteReason = &deleteReason.String
}
}
if md5Hash != nil && len(md5Hash) == 16 {
object.MD5HashBytes = md5Hash
md5String := hex.EncodeToString(md5Hash)
object.MD5Hash = &md5String
}
if sha256Hash != nil && len(sha256Hash) == 32 {
object.SHA256HashBytes = sha256Hash
sha256String := hex.EncodeToString(sha256Hash)
object.SHA256Hash = &sha256String
}
object.ObjectType = objectType
return object, nil
}
lib/db/sql.go 0 → 100644
View file @ 3d53bc53
package db
var selectObjectByBucketKey = `
SELECT
content_type,
dest_url,
"type",
deleted_at,
delete_reason,
md5_hash,
sha256_hash
FROM
objects
WHERE
bucket_key = $1
LIMIT 1
`
metrics/collector.go → lib/metrics/collector.go
View file @ 3d53bc53
......@@ -7,8 +7,8 @@ import (
"net"
"strings"
log "github.com/Sirupsen/logrus"
"github.com/oschwald/maxminddb-golang"
"github.com/rs/zerolog/log"
"gopkg.in/olivere/elastic.v5"
"gopkg.in/olivere/elastic.v5/config"
)
......@@ -95,7 +95,7 @@ func New(elasticURL string, maxmindLoc string, enableHostnameWhitelist bool, hos
return nil, fmt.Errorf("failed to parse elasticURL: %s", err)
}
if cfg.Index == "" {
log.Info(`empty index name in elasticURL, using "cdn-origin_requests"`)
log.Info().Msg(`empty index name in elasticURL, using "cdn-origin_requests"`)
cfg.Index = "cdn-origin_requests"
}
......@@ -109,11 +109,7 @@ func New(elasticURL string, maxmindLoc string, enableHostnameWhitelist bool, hos
if err != nil {
return nil, fmt.Errorf("failed to ping Elasticsearch server: %s", err)
}
// TODO: if code != 200
log.WithFields(log.Fields{
"statusCode": code,
"version": info.Version.Number,
}).Debug("elasticsearch ping success")
log.Debug().Int("statusCode", code).Str("version", info.Version.Number).Msg("elasticsearch ping success")
// Check if the index exists or create it
exists, err := client.IndexExists(cfg.Index).Do(ctx)
......@@ -121,7 +117,7 @@ func New(elasticURL string, maxmindLoc string, enableHostnameWhitelist bool, hos
return nil, fmt.Errorf("failed to determine if the index exists: %s", err)
}
if !exists {
log.WithField("index", cfg.Index).Info("creating Elasticsearch index")
log.Info().Str("index", cfg.Index).Msg("creating Elasticsearch index")
createIndex, err := client.CreateIndex(cfg.Index).
BodyString(mapping).
Do(ctx)
......@@ -141,7 +137,7 @@ func New(elasticURL string, maxmindLoc string, enableHostnameWhitelist bool, hos
return nil, fmt.Errorf("failed to determine if the timestamp pipeline exists: %s", err)
}
if len(pipelines) == 0 {
log.WithField("pipeline", "timestamp").Info("creating Elasticsearch ingest pipeline")
log.Info().Str("pipeline", "timestamp").Msg("creating Elasticsearch ingest pipeline")
putPipeline, err := elastic.NewIngestPutPipelineService(client).
Id("timestamp").
BodyString(timestampPipeline).
......
metrics/geoip.go → lib/metrics/geoip.go
View file @ 3d53bc53
File moved
metrics/match.go → lib/metrics/match.go
View file @ 3d53bc53
File moved
metrics/record.go → lib/metrics/record.go
View file @ 3d53bc53
File moved
lib/thumbnailer/cache.go 0 → 100644
View file @ 3d53bc53
package thumbnailer
import (
"io"
"os"
"path/filepath"
)
// ThumbnailCache allows access to thumbnails stored in a directory. Each
// thumbnail has a key, which uniquely identifies it. The key should be a unique
// ID from a database or the original file's hash.
type ThumbnailCache struct {
Directory string
ThumbnailerURL string
}
// NewThumbnailCache creates a new *ThumbnailCache.
func NewThumbnailCache(directory, thumbnailerURL string) *ThumbnailCache {
return &ThumbnailCache{
Directory: directory,
ThumbnailerURL: thumbnailerURL,
}
}
// GetThumbnail returns a thumbnail that is cached. If no cached copy exists, a
// exists, a NoCachedCopy error is returned.
func (c *ThumbnailCache) GetThumbnail(key string) (io.ReadCloser, error) {
if key == "" {
return nil, NoKeySpecified
}
path := filepath.Join(c.Directory, key)
data, err := os.Open(path)
if os.IsNotExist(err) {
return nil, NoCachedCopy
}
return data, err
}
// SetThumbnail stores a thumbnail with the specified key.
func (c *ThumbnailCache) SetThumbnail(key string, data io.Reader) error {
if key == "" {
return NoKeySpecified
}
path := filepath.Join(c.Directory, key)
file, err := os.Create(path)
if err != nil {
return err
}
defer file.Close()
_, err = io.Copy(file, data)
if err != nil {
_ = os.Remove(path)
}
return err
}
// Transform generates a thumbnail and caches it.
func (c *ThumbnailCache) Transform(key string, contentType string, data io.Reader) error {
if key == "" {
return NoKeySpecified
}
outputImage, err := Transform(c.ThumbnailerURL, contentType, data)
if err != nil {
return err
}
return c.SetThumbnail(key, outputImage)
}
lib/thumbnailer/errors.go 0 → 100644
View file @ 3d53bc53
package thumbnailer
type thumbnailerError struct {
Err string
}
// Error implements error.
func (e *thumbnailerError) Error() string {
return e.Err
}
// NoCachedCopy means there is no cached copy for the specified key available.
var NoCachedCopy error = &thumbnailerError{"no cached copy of the thumbnail requested is available"}
// InputTooLarge means that the pixel size of the input image is too big to be thumbnailed.
var InputTooLarge error = &thumbnailerError{"the input size in pixels is too large"}
// NoKeySpecified means that no key was specified.
var NoKeySpecified error = &thumbnailerError{"no key specified"}