.gitignore

@@ -15,12 +15,15 @@
 
 # Built package binary
 cdn-origin
+main
 
 # Package configuration file (not including sample configuration)
-cdn-origin.toml
+config.toml
+config.prod.toml
 
 # MaxMind GeoLite2 Country database files
 GeoLite2-Country.mmdb
 
 # JetBrains .idea
 .idea/
+

Dockerfile

-FROM golang:alpine
+FROM golang:1.21-alpine
 
-COPY cdn-origin.go $GOPATH/src/github.com/whats-this/cdn-origin/
-COPY prometheus/ $GOPATH/src/github.com/whats-this/cdn-origin/prometheus/
-COPY weed/ $GOPATH/src/github.com/whats-this/cdn-origin/weed/
+RUN apk add --no-cache --virtual .build-deps git build-base
 
-RUN apk add --no-cache --virtual .build-deps git && \
+COPY go.mod /git/owo.codes/whats-this/cdn-origin/
+COPY go.sum /git/owo.codes/whats-this/cdn-origin/
+COPY main.go /git/owo.codes/whats-this/cdn-origin/
+COPY lib /git/owo.codes/whats-this/cdn-origin/lib
 
-    go-wrapper download github.com/lib/pq && \
-    go-wrapper download github.com/prometheus/client_golang/prometheus && \
-    go-wrapper download github.com/Sirupsen/logrus && \
-    go-wrapper download github.com/spf13/pflag && \
-    go-wrapper download github.com/spf13/viper && \
-    go-wrapper download github.com/valyala/fasthttp && \
-
-    go-wrapper install github.com/whats-this/cdn-origin && \
+RUN cd /git/owo.codes/whats-this/cdn-origin && \
+    go mod download && \
+    go build main.go && \
     apk del .build-deps
 
-WORKDIR $GOPATH/src/github.com/whats-this/cdn-origin
-ENTRYPOINT ["go-wrapper", "run"]
+WORKDIR /git/owo.codes/whats-this/cdn-origin
+ENTRYPOINT ["./main"]
+

LICENSE

 The MIT License (MIT)
 
-Copyright (c) 2017 Dean Sheather, whats-this
+Copyright (c) 2019 Dean Sheather, whats-this
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

README.md

-# Whats-This CDN Origin
+# cdn-origin
 
 Simple but quick Golang webserver that serves requests to get files and
-redirects from a [PostgreSQL](https://www.postgresql.org) and
-[SeaweedFS](https://github.com/chrislusf/seaweedfs) backend.
+redirects from [PostgreSQL](https://www.postgresql.org).
+
+### Features
+- Serves files, short URLs and "tombstones" (deleted file markers)
+- Allows for URL previewing on short URLs (add `?preview`)
+- Allows for thumbnail generation on images via external thumbnailer service (if
+  enabled, add `?thumbnail`)
+- Can be configured to store generalized metrics
 
 ### Requirements
 
 - PostgreSQL server with `objects` table
-- SeaweedFS cluster with files in the `objects` table
+- Access to the folder where the files are stored
+- If metrics support is desired, you will need an Elasticsearch server setup as
+  noted below
+- If thumbnail support is desired, you will need a webserver with an endpoint
+  that returns a thumbnail from raw POSTed image data (`jpeg`, `gif`, `png`,
+  `webp`) such as
+  [thumbnail-service](https://owo.codes/whats-this/thumbnail-service). We
+  recommend that the service outputs 200x200 (max) JPEG images
 
 ### Usage
 
 ```
-$ go get -u github.com/whats-this/cdn-origin
-
-# With configuration file
-$ cp $GOPATH/src/github.com/whats-this/cdn-origin/cdn-origin.sample.toml /etc/cdn-origin/cdn-origin.toml
-$ vim /etc/cdn-origin/cdn-origin.toml
-$ cdn-origin
-
-# With environment variables
-$ set DATABASE_CONNECTION_URL="postgres://postgres@localhost/data?sslmode=disable"
-$ set SEAWEED_MASTER_URL="http://localhost:9333"
-$ ...
-$ cdn-origin
-
-# With flags
-$ cdn-origin \
-    --database-connection-url="postgres://postgres@localhost/data?sslmode=disable" \
-    --seaweed-master-url="http://localhost:9333" \
-    ...
-
-# Flags take precedence over environment variables, which take precedence over config files
+$ git clone https://owo.codes/whats-this/cdn-origin.git
+$ cd cdn-origin
+$ cp config.sample.toml config.toml
+$ vim config.toml
+$ go build main.go
+$ ./main --config-file "./config.toml"
 ```
 
-Information about configuration variables and their purpose can be found in
-[cdn-origin.sample.toml](cdn-origin.sample.toml). Configuration is handled by
-[Viper](https://github.com/spf13/viper).
-
 ### Metrics
 
 If `metrics.enable` is `true`, request metadata will be indexed in the provided
@@ -55,15 +50,12 @@ Elaticsearch server in the following format:
 
 The index and `@timestamp` pipeline are created automatically if `cdn-origin`
 has permission. Alternatively, the mapping and pipeline can be created by other
-means using the `.json` files in [metrics/](metrics).
+means using the `.json` files in [lib/metrics/](lib/metrics).
 
 ### TODO
 
-- [ ] Process chunked files stored in SeaweedFS, similar to [how SeaweedFS cli
-  handles it](https://github.com/chrislusf/seaweedfs/wiki/Large-File-Handling)
-- [ ] Add TTL to volume cache
-- [ ] Write tests
-- [ ] Add thumbnail functionality (SeaweedFS supports this)
+- `OPTIONS`/`HEAD` support
+- Write tests
 
 ### License
 

cdn-origin.go

-package main
-
-import (
-	"database/sql"
-	"fmt"
-	"html/template"
-	"net"
-	"strings"
-	"time"
-
-	"github.com/whats-this/cdn-origin/metrics"
-	"github.com/whats-this/cdn-origin/weed"
-
-	log "github.com/Sirupsen/logrus"
-	_ "github.com/lib/pq"
-	"github.com/spf13/pflag"
-	"github.com/spf13/viper"
-	"github.com/valyala/fasthttp"
-)
-
-// version is the current version of cdn-origin.
-const version = "0.4.0"
-
-// Default SeaweedFS fetch query parameters.
-const defaultSeaweedFSQueryParameters = ""
-
-// HTTP header strings.
-const (
-	accept   = "accept"
-	host     = "host"
-	location = "Location"
-)
-
-// redirectHTML is the html/template template for generating redirect HTML.
-const redirectHTML = `<html><head><meta charset="UTF-8" /><meta http-equiv=refresh content="0; url={{.}}" /><script type="text/javascript">window.location.href="{{.}}"</script><title>Redirect</title></head><body><p>If you are not redirected automatically, click <a href="{{.}}">here</a> to go to the destination.</p></body></html>`
-
-var redirectHTMLTemplate *template.Template
-
-// redirectPreviewHTML is the html/template template for generating redirect preview HTML.
-const redirectPreviewHTML = `<html><head><meta charset="UTF-8" /><title>Redirect Preview</title></head><body><p>This link goes to <code>{{.}}</code>. If you would like to visit this link, click <a href="{{.}}">here</a> to go to the destination.</p></body></html>`
-
-var redirectPreviewHTMLTemplate *template.Template
-
-func init() {
-	// Read in configuration
-	flags := pflag.NewFlagSet("cdn-origin", pflag.ExitOnError)
-
-	// database.connectionURL* (string): PostgreSQL connection URL
-	flags.String("database-connection-url", "", "* PostgreSQL connection URL")
-	viper.BindPFlag("database.connectionURL", flags.Lookup("database-connection-url"))
-	viper.BindEnv("database.connectionURL", "DATABASE_CONNECTION_URL")
-
-	// debug (bool=false): enable debug mode (logs requests and prints other information)
-	flags.Bool("debug", false, "Enable debug mode (logs requests and prints other information)")
-	viper.BindPFlag("debug", flags.Lookup("debug"))
-	viper.BindEnv("debug", "DEBUG")
-
-	// http.compressResponse (bool=false): enable transparent response compression
-	flags.Bool("compress-response", false, "Enable transparent response compression")
-	viper.BindPFlag("http.compressResponse", flags.Lookup("compress-response"))
-	viper.BindEnv("http.compressResponse", "HTTP_COMPRESS_RESPONSE")
-
-	// http.listenAddress (string=":8080"): TCP address to listen to for HTTP requests
-	flags.String("listen-address", ":8080", "TCP address to listen to for HTTP requests")
-	viper.BindPFlag("http.listenAddress", flags.Lookup("listen-address"))
-	viper.BindEnv("http.listenAddress", "HTTP_LISTEN_ADDRESS")
-
-	// http.trustProxy (bool=false): trust X-Forwarded-For header from proxy
-	flags.Bool("trust-proxy", false, "Trust X-Forwarded-For header from proxy")
-	viper.BindPFlag("http.trustProxy", flags.Lookup("trust-proxy"))
-	viper.BindEnv("http.trustProxy", "HTTP_TRUST_PROXY")
-
-	// metrics.enable (bool=false): enable anonymized request recording (country code, hostname, object type, status
-	// code)
-	flags.Bool("enable-metrics", false,
-		"Enable anonymized request recording (status code, country code, object type, hostname)")
-	viper.BindPFlag("metrics.enable", flags.Lookup("enable-metrics"))
-	viper.BindEnv("metrics.enable", "ENABLE_METRICS")
-
-	// metrics.elasticURL (string): Elasticsearch URL to connect to for metrics, required if metrics.enable == true
-	flags.String("elastic-url", "", "Elastic URL to connect to for metrics")
-	viper.BindPFlag("metrics.elasticURL", flags.Lookup("elastic-url"))
-	viper.BindEnv("metrics.elasticURL", "ELASTIC_URL")
-
-	// metrics.enableHostnameWhitelist (bool=false): enable hostname whitelist for metrics
-	// (metrics.hostnameWhitelist)
-	flags.Bool("metrics-enable-whitelist", false, "Enable hostname whitelist for metrics")
-	viper.BindPFlag("metrics.enableHostnameWhitelist", flags.Lookup("metrics-enable-whitelist"))
-	viper.BindEnv("metrics.enableHostnameWhitelist", "METRICS_ENABLE_WHITELIST")
-
-	// metrics.hostnameWhitelist (string[]=[]): hostnames to whitelist for metrics (config file only)
-
-	// metrics.maxmindDBLocation (string): location of MaxMind GeoLite2 Country database file (.mmdb) on disk, if
-	// empty country codes will not be recorded by metrics collector
-	flags.String("maxmind-db-location", "", "Location of MaxMind GeoLite2 Country database on disk")
-	viper.BindPFlag("metrics.maxmindDBLocation", flags.Lookup("maxmind-db-location"))
-	viper.BindEnv("metrics.maxmindDBLocation", "MAXMIND_DB_LOCATION")
-
-	// seaweed.masterURL* (string): SeaweedFS master URL
-	flags.String("seaweed-master-url", "", "* SeaweedFS master URL")
-	viper.BindPFlag("seaweed.masterURL", flags.Lookup("seaweed-master-url"))
-	viper.BindEnv("seaweed.masterURL", "SEAWEED_MASTER_URL")
-
-	// Configuration file settings
-	viper.SetConfigType("toml")
-	viper.SetConfigName("cdn-origin")
-	viper.AddConfigPath(".")
-	viper.AddConfigPath("/etc/cdn-origin/")
-
-	if err := viper.ReadInConfig(); err != nil {
-		if _, ok := err.(viper.ConfigFileNotFoundError); !ok {
-			log.WithError(err).Fatal("failed to read in configuration")
-		}
-	}
-
-	// Enable debug mode
-	if viper.GetBool("debug") {
-		log.SetLevel(log.DebugLevel)
-	}
-
-	// Print configuration variables to debug
-	log.WithFields(log.Fields{
-		"database.connectionURL":          viper.GetString("database.connectionURL"),
-		"debug":                           viper.GetBool("debug"),
-		"http.compressResponse":           viper.GetBool("http.compressResponse"),
-		"http.listenAddress":              viper.GetString("http.listenAddress"),
-		"metrics.enable":                  viper.GetBool("metrics.enable"),
-		"metrics.elasticURL":              viper.GetString("metrics.elasticURL"),
-		"len(metrics.hostnameWhitelist)":  len(viper.GetStringSlice("metrics.hostnameWhitelist")),
-		"metrics.enableHostnameWhitelist": viper.GetString("metrics.enableHostnameWhitelist"),
-		"seaweed.masterURL":               viper.GetString("seaweed.masterURL"),
-	}).Debug("retrieved configuration")
-
-	// Ensure required configuration variables are set
-	if viper.GetString("database.connectionURL") == "" {
-		log.Fatal("database.connectionURL is required")
-	}
-	if viper.GetString("seaweed.masterURL") == "" {
-		log.Fatal("seaweed.masterURL is required")
-	}
-	if viper.GetBool("metrics.enable") && viper.GetString("metrics.elasticURL") == "" {
-		log.Fatal("metrics.elasticURL is required when metrics are enabled")
-	}
-
-	// Parse redirect templates
-	var err error
-	redirectHTMLTemplate, err = template.New("redirectHTML").Parse(redirectHTML)
-	if err != nil {
-		log.WithError(err).Fatal("failed to parse redirectHTML template")
-	}
-	redirectPreviewHTMLTemplate, err = template.New("redirectPreviewHTML").Parse(redirectPreviewHTML)
-	if err != nil {
-		log.WithError(err).Fatal("failed to parse redirectPreviewHTML template")
-	}
-}
-
-var db *sql.DB
-var collector *metrics.Collector
-var seaweed *weed.Seaweed
-
-func main() {
-	var err error
-
-	// Attempt to connect to SeaweedFS master
-	seaweed = weed.New(viper.GetString("seaweed.masterURL"), time.Second*5)
-	err = seaweed.Ping()
-	if err != nil {
-		log.WithError(err).Fatal("failed to ping SeaweedFS master")
-	}
-
-	// Connect to PostgreSQL database
-	// TODO: abstractify database connection
-	// TODO: create table if it doesn't exist
-	db, err = sql.Open("postgres", viper.GetString("database.connectionURL"))
-	if err != nil {
-		log.WithError(err).Fatal("failed to open database connection")
-	}
-
-	// Setup metrics collector
-	if viper.GetBool("metrics.enable") {
-		hostnameWhitelist := []string{}
-		if viper.GetBool("metrics.enableHostnameWhitelist") {
-			switch w := viper.Get("metrics.hostnameWhitelist").(type) {
-			case []interface{}:
-				for _, s := range w {
-					hostnameWhitelist = append(hostnameWhitelist, strings.TrimSpace(fmt.Sprint(s)))
-				}
-				break
-			default:
-				log.Fatal("metrics.hostnameWhitelist is not an array")
-			}
-		}
-		collector, err = metrics.New(
-			viper.GetString("metrics.elasticURL"),
-			viper.GetString("metrics.maxmindDBLocation"),
-			viper.GetBool("metrics.enableHostnameWhitelist"),
-			hostnameWhitelist,
-		)
-		if err != nil {
-			log.WithError(err).Fatal("failed to setup metrics collector")
-		}
-	}
-
-	// Launch server
-	h := requestHandler
-	if viper.GetBool("http.compressResponse") {
-		h = fasthttp.CompressHandler(h)
-	}
-	log.Info("Attempting to listen on " + viper.GetString("http.listenAddress"))
-	server := &fasthttp.Server{
-		Handler:                       h,
-		Name:                          "whats-this/cdn-origin v" + version,
-		ReadBufferSize:                1024 * 6, // 6 KB
-		ReadTimeout:                   time.Minute * 30,
-		WriteTimeout:                  time.Minute * 30,
-		GetOnly:                       true, // TODO: OPTIONS/HEAD requests
-		LogAllErrors:                  log.GetLevel() == log.DebugLevel,
-		DisableHeaderNamesNormalizing: false,
-		Logger: log.New(),
-	}
-	if err := server.ListenAndServe(viper.GetString("http.listenAddress")); err != nil {
-		log.WithError(err).Fatal("error in server.ListenAndServe")
-	}
-}
-
-func recordMetrics(ctx *fasthttp.RequestCtx, objectType string) {
-	if !viper.GetBool("metrics.enable") {
-		return
-	}
-
-	var remoteIP net.IP
-	if viper.GetBool("http.trustProxy") {
-		ipString := string(ctx.Request.Header.Peek("X-Forwarded-For"))
-		remoteIP = net.ParseIP(strings.Split(ipString, ",")[0])
-	} else {
-		remoteIP = ctx.RemoteIP()
-	}
-
-	remoteIP = net.ParseIP("51.15.83.140")
-
-	hostBytes := ctx.Request.Header.Peek(host)
-	if len(hostBytes) != 0 {
-		go func() {
-			// Check hostname
-			hostStr, isValid := collector.MatchHostname(string(hostBytes))
-			if !isValid {
-				return
-			}
-
-			// Get country code of visitor
-			countryCode, err := collector.GetCountryCode(remoteIP)
-			if err != nil {
-				// Don't log the error here, it might contain an IP address
-				log.Warn("failed to get country code for IP, omitting from record")
-			}
-
-			record := metrics.GetRecord()
-			record.CountryCode = countryCode
-			record.Hostname = hostStr
-			record.ObjectType = objectType
-			record.StatusCode = ctx.Response.StatusCode()
-			err = collector.Put(record)
-			if err != nil {
-				log.WithError(err).Warn("failed to collect record")
-			}
-			log.Debug("successfully collected metrics")
-		}()
-	}
-}
-
-func requestHandler(ctx *fasthttp.RequestCtx) {
-	metricsObjectType := ""
-
-	// Log requests in debug mode, wrapped in an if statement to prevent unnecessary memory allocations
-	if log.GetLevel() == log.DebugLevel {
-		log.WithFields(log.Fields{
-			"connRequestNumber": ctx.ConnRequestNum(),
-			"method":            string(ctx.Method()),
-			"queryString":       ctx.QueryArgs(),
-			"remoteIP":          ctx.RemoteIP(),
-			"requestURI":        string(ctx.RequestURI()),
-		}).Debug("request received")
-	}
-
-	// Fetch object from database
-	var backendFileID sql.NullString
-	var contentType sql.NullString
-	var destURL sql.NullString
-	var objectType int
-	err := db.QueryRow(
-		`SELECT backend_file_id, content_type, dest_url, "type" FROM objects WHERE bucket_key=$1 LIMIT 1`,
-		fmt.Sprintf("public%s", ctx.Path()),
-	).Scan(&backendFileID, &contentType, &destURL, &objectType)
-	switch {
-	case err == sql.ErrNoRows:
-		ctx.SetStatusCode(fasthttp.StatusNotFound)
-		ctx.SetContentType("text/plain; charset=utf8")
-		fmt.Fprintf(ctx, "404 Not Found: %s", ctx.Path())
-		recordMetrics(ctx, metricsObjectType)
-		return
-	case err != nil:
-		log.WithError(err).Error("failed to run SELECT query on database")
-		ctx.SetStatusCode(fasthttp.StatusInternalServerError)
-		ctx.SetContentType("text/plain; charset=utf8")
-		fmt.Fprint(ctx, "500 Internal Server Error")
-		ctx.SetUserValue("object_type", "file")
-		recordMetrics(ctx, metricsObjectType)
-		return
-	}
-
-	switch objectType {
-	case 0: // file
-		metricsObjectType = "file"
-
-		// Get object from SeaweedFS and write to response
-		if !backendFileID.Valid {
-			log.WithField("bucket_key", string(ctx.Path())).Warn("found file object with NULL backend_file_id")
-			ctx.SetStatusCode(fasthttp.StatusInternalServerError)
-			ctx.SetContentType("text/plain; charset=utf8")
-			fmt.Fprint(ctx, "500 Internal Server Error")
-			recordMetrics(ctx, metricsObjectType)
-			return
-		}
-
-		// TODO: ?thumbnail query parameter for images
-		statusCode, contentSize, err := seaweed.Get(ctx, backendFileID.String, defaultSeaweedFSQueryParameters)
-		if err != nil {
-			log.WithError(err).Warn("failed to retrieve file from SeaweedFS volume server")
-			ctx.SetStatusCode(fasthttp.StatusInternalServerError)
-			ctx.SetContentType("text/plain; charset=utf8")
-			fmt.Fprint(ctx, "500 Internal Server Error")
-			recordMetrics(ctx, metricsObjectType)
-			return
-		}
-		if statusCode != fasthttp.StatusOK {
-			log.WithFields(log.Fields{
-				"expected": fasthttp.StatusOK,
-				"got":      statusCode,
-			}).Warn("unexpected status code while retrieving file from SeaweedFS volume server")
-			ctx.SetStatusCode(fasthttp.StatusInternalServerError)
-			ctx.SetContentType("text/plain; charset=utf8")
-			fmt.Fprint(ctx, "500 Internal Server Error")
-			recordMetrics(ctx, metricsObjectType)
-			return
-		}
-
-		if contentType.Valid {
-			ctx.SetContentType(contentType.String)
-		} else {
-			ctx.SetContentType("application/octet-stream")
-		}
-		ctx.Response.Header.SetContentLength(contentSize)
-		recordMetrics(ctx, metricsObjectType)
-
-	case 1: // redirect
-		metricsObjectType = "redirect"
-
-		if !destURL.Valid {
-			log.Warn("encountered redirect object with NULL dest_url")
-			ctx.SetStatusCode(fasthttp.StatusInternalServerError)
-			ctx.SetContentType("text/plain; charset=utf8")
-			fmt.Fprint(ctx, "500 Internal Server Error")
-			recordMetrics(ctx, metricsObjectType)
-			return
-		}
-
-		previewMode := ctx.QueryArgs().Has("preview")
-		var err error
-		if previewMode {
-			err = redirectPreviewHTMLTemplate.Execute(ctx, destURL.String)
-		} else {
-			err = redirectHTMLTemplate.Execute(ctx, destURL.String)
-		}
-
-		if err != nil {
-			log.WithFields(log.Fields{
-				"error":    err,
-				"dest_url": destURL.String,
-				"preview":  ctx.QueryArgs().Has("preview"),
-			}).Warn("failed to generate HTML redirect page to send to client")
-			ctx.SetContentType("text/plain; charset=utf8")
-			fmt.Fprintf(ctx, "Failed to generate HTML redirect page, destination URL: %s", destURL.String)
-			recordMetrics(ctx, metricsObjectType)
-			return
-		}
-
-		ctx.SetContentType("text/html; charset=ut8")
-		if !previewMode {
-			ctx.SetStatusCode(fasthttp.StatusFound)
-			ctx.Response.Header.Set(location, destURL.String)
-		}
-		recordMetrics(ctx, metricsObjectType)
-	}
-}

cdn-origin.sample.toml → config.sample.toml

-# Enable debug mode (logs requests and prints other information)
-debug = false
+[log]
+    # Log level (5=panic, 4=fatal, 3=error, 2=warn, 1=info, 0=debug)
+    level = 1
 
 [database]
     # PostgreSQL connection URL, see
@@ -7,6 +8,9 @@ debug = false
     # more information
     connectionURL = "postgres://postgres:password@localhost/whats_this?sslmode=disable"
 
+    # Bucket to serve objects from
+    objectBucket = "public"
+
 [http]
     # Enable transparent response compression (only when the client Accepts it)
     compressResponse = false
@@ -49,7 +53,21 @@ debug = false
     # code data will be collected. https://dev.maxmind.com/geoip/geoip2/geolite2
     maxmindDBLocation = "/var/data/maxmind/GeoLite2-Country.mmdb"
 
-[seaweed]
-    # SeaweedFS master URL (used for looking up volumes, volumes must be
-    # accessible by their publicUrl)
-    masterURL = "http://localhost:9333"
+[files]
+    # Storage location of the bucket on disk
+    storageLocation = "/var/data/buckets/public"
+
+[thumbnails]
+    # Enable thumbnails? (add ?thumbnail to the end of a file object URL)
+    enable = true
+
+    # Thumbnailer URL. This is a endpoint that accepts a POST request containing
+    # raw image data and returns a thumbnail. For example, see
+    # https://owo.codes/whats-this/thumbnail-service.
+    thumbnailerURL = "http://localhost:8081/thumbnail"
+
+    # Enable thumbnail cache?
+    cacheEnable = true
+
+    # Thumbnail cache location (if enabled).
+    cacheLocation = "/tmp/thumbs"

go.mod

+module owo.codes/whats-this/cdn-origin
+
+go 1.21
+
+require (
+	github.com/lib/pq v1.10.9
+	github.com/oschwald/maxminddb-golang v1.12.0
+	github.com/pkg/errors v0.9.1
+	github.com/rs/zerolog v1.31.0
+	github.com/spf13/pflag v1.0.5
+	github.com/spf13/viper v1.17.0
+	github.com/valyala/fasthttp v1.50.0
+	gopkg.in/olivere/elastic.v5 v5.0.86
+)
+
+require (
+	github.com/andybalholm/brotli v1.0.6 // indirect
+	github.com/fsnotify/fsnotify v1.7.0 // indirect
+	github.com/hashicorp/hcl v1.0.0 // indirect
+	github.com/josharian/intern v1.0.0 // indirect
+	github.com/klauspost/compress v1.17.2 // indirect
+	github.com/magiconair/properties v1.8.7 // indirect
+	github.com/mailru/easyjson v0.7.7 // indirect
+	github.com/mattn/go-colorable v0.1.13 // indirect
+	github.com/mattn/go-isatty v0.0.20 // indirect
+	github.com/mitchellh/mapstructure v1.5.0 // indirect
+	github.com/pelletier/go-toml/v2 v2.1.0 // indirect
+	github.com/sagikazarmark/locafero v0.3.0 // indirect
+	github.com/sagikazarmark/slog-shim v0.1.0 // indirect
+	github.com/sourcegraph/conc v0.3.0 // indirect
+	github.com/spf13/afero v1.10.0 // indirect
+	github.com/spf13/cast v1.5.1 // indirect
+	github.com/spf13/jwalterweatherman v1.1.0 // indirect
+	github.com/subosito/gotenv v1.6.0 // indirect
+	github.com/valyala/bytebufferpool v1.0.0 // indirect
+	go.uber.org/atomic v1.11.0 // indirect
+	go.uber.org/multierr v1.11.0 // indirect
+	golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa // indirect
+	golang.org/x/sys v0.14.0 // indirect
+	golang.org/x/text v0.14.0 // indirect
+	gopkg.in/ini.v1 v1.67.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+)

lib/db/connect.go

+package db
+
+import (
+	"database/sql"
+
+	// postgres driver for database/sql
+	_ "github.com/lib/pq"
+)
+
+// DB is the current database connection.
+var DB *sql.DB
+
+// Connect to the database using the given driver and connection URL.
+func Connect(driver string, connectionURL string) error {
+	var err error
+	DB, err = sql.Open(driver, connectionURL)
+	if err != nil {
+		return err
+	}
+	return DB.Ping()
+}

lib/db/models.go

+package db
+
+import (
+	"time"
+)
+
+// Object represents a partial object from the database.
+type Object struct {
+	ContentType     *string    `json:"content_type"`
+	DestURL         *string    `json:"dest_url"`
+	ObjectType      int        `json:"object_type"`
+	DeletedAt       *time.Time `json:"deleted_at"`
+	DeleteReason    *string    `json:"delete_reason"`
+	MD5HashBytes    []byte     `json:"-"`
+	SHA256HashBytes []byte     `json:"-"`
+
+	// Computed fields
+	MD5Hash    *string `json:"md5_hash"`
+	SHA256Hash *string `json:"sha256_hash"`
+}

lib/db/queries.go

+package db
+
+import (
+	"database/sql"
+	"encoding/hex"
+	"fmt"
+
+	"github.com/lib/pq"
+)
+
+// SelectObjectByBucketKey returns an object from a bucket and a key.
+func SelectObjectByBucketKey(bucket, key string) (Object, error) {
+	var object Object
+
+	var contentType sql.NullString
+	var destURL sql.NullString
+	var objectType int
+	var deletedAt pq.NullTime
+	var deleteReason sql.NullString
+	var md5Hash []byte
+	var sha256Hash []byte
+	err := DB.QueryRow(selectObjectByBucketKey, fmt.Sprintf("%s/%s", bucket, key)).
+		Scan(&contentType, &destURL, &objectType, &deletedAt, &deleteReason, &md5Hash, &sha256Hash)
+	if err != nil {
+		return object, err
+	}
+
+	// Populate object values
+	if contentType.Valid {
+		object.ContentType = &contentType.String
+	}
+	if destURL.Valid {
+		object.DestURL = &destURL.String
+	}
+	if deletedAt.Valid {
+		object.DeletedAt = &deletedAt.Time
+		if deleteReason.Valid {
+			object.DeleteReason = &deleteReason.String
+		}
+	}
+	if md5Hash != nil && len(md5Hash) == 16 {
+		object.MD5HashBytes = md5Hash
+		md5String := hex.EncodeToString(md5Hash)
+		object.MD5Hash = &md5String
+	}
+	if sha256Hash != nil && len(sha256Hash) == 32 {
+		object.SHA256HashBytes = sha256Hash
+		sha256String := hex.EncodeToString(sha256Hash)
+		object.SHA256Hash = &sha256String
+	}
+	object.ObjectType = objectType
+	return object, nil
+}

lib/db/sql.go

+package db
+
+var selectObjectByBucketKey = `
+SELECT
+	content_type,
+	dest_url,
+	"type",
+	deleted_at,
+	delete_reason,
+	md5_hash,
+	sha256_hash
+FROM
+	objects
+WHERE
+	bucket_key = $1
+LIMIT 1
+`

metrics/collector.go → lib/metrics/collector.go

@@ -7,8 +7,8 @@ import (
 	"net"
 	"strings"
 
-	log "github.com/Sirupsen/logrus"
 	"github.com/oschwald/maxminddb-golang"
+	"github.com/rs/zerolog/log"
 	"gopkg.in/olivere/elastic.v5"
 	"gopkg.in/olivere/elastic.v5/config"
 )
@@ -95,7 +95,7 @@ func New(elasticURL string, maxmindLoc string, enableHostnameWhitelist bool, hos
 		return nil, fmt.Errorf("failed to parse elasticURL: %s", err)
 	}
 	if cfg.Index == "" {
-		log.Info(`empty index name in elasticURL, using "cdn-origin_requests"`)
+		log.Info().Msg(`empty index name in elasticURL, using "cdn-origin_requests"`)
 		cfg.Index = "cdn-origin_requests"
 	}
 
@@ -109,11 +109,7 @@ func New(elasticURL string, maxmindLoc string, enableHostnameWhitelist bool, hos
 	if err != nil {
 		return nil, fmt.Errorf("failed to ping Elasticsearch server: %s", err)
 	}
-	// TODO: if code != 200
-	log.WithFields(log.Fields{
-		"statusCode": code,
-		"version":    info.Version.Number,
-	}).Debug("elasticsearch ping success")
+	log.Debug().Int("statusCode", code).Str("version", info.Version.Number).Msg("elasticsearch ping success")
 
 	// Check if the index exists or create it
 	exists, err := client.IndexExists(cfg.Index).Do(ctx)
@@ -121,7 +117,7 @@ func New(elasticURL string, maxmindLoc string, enableHostnameWhitelist bool, hos
 		return nil, fmt.Errorf("failed to determine if the index exists: %s", err)
 	}
 	if !exists {
-		log.WithField("index", cfg.Index).Info("creating Elasticsearch index")
+		log.Info().Str("index", cfg.Index).Msg("creating Elasticsearch index")
 		createIndex, err := client.CreateIndex(cfg.Index).
 			BodyString(mapping).
 			Do(ctx)
@@ -141,7 +137,7 @@ func New(elasticURL string, maxmindLoc string, enableHostnameWhitelist bool, hos
 		return nil, fmt.Errorf("failed to determine if the timestamp pipeline exists: %s", err)
 	}
 	if len(pipelines) == 0 {
-		log.WithField("pipeline", "timestamp").Info("creating Elasticsearch ingest pipeline")
+		log.Info().Str("pipeline", "timestamp").Msg("creating Elasticsearch ingest pipeline")
 		putPipeline, err := elastic.NewIngestPutPipelineService(client).
 			Id("timestamp").
 			BodyString(timestampPipeline).

lib/thumbnailer/cache.go

+package thumbnailer
+
+import (
+	"io"
+	"os"
+	"path/filepath"
+)
+
+// ThumbnailCache allows access to thumbnails stored in a directory. Each
+// thumbnail has a key, which uniquely identifies it. The key should be a unique
+// ID from a database or the original file's hash.
+type ThumbnailCache struct {
+	Directory      string
+	ThumbnailerURL string
+}
+
+// NewThumbnailCache creates a new *ThumbnailCache.
+func NewThumbnailCache(directory, thumbnailerURL string) *ThumbnailCache {
+	return &ThumbnailCache{
+		Directory:      directory,
+		ThumbnailerURL: thumbnailerURL,
+	}
+}
+
+// GetThumbnail returns a thumbnail that is cached. If no cached copy exists, a
+// exists, a NoCachedCopy error is returned.
+func (c *ThumbnailCache) GetThumbnail(key string) (io.ReadCloser, error) {
+	if key == "" {
+		return nil, NoKeySpecified
+	}
+	path := filepath.Join(c.Directory, key)
+	data, err := os.Open(path)
+	if os.IsNotExist(err) {
+		return nil, NoCachedCopy
+	}
+	return data, err
+}
+
+// SetThumbnail stores a thumbnail with the specified key.
+func (c *ThumbnailCache) SetThumbnail(key string, data io.Reader) error {
+	if key == "" {
+		return NoKeySpecified
+	}
+	path := filepath.Join(c.Directory, key)
+	file, err := os.Create(path)
+	if err != nil {
+		return err
+	}
+	defer file.Close()
+	_, err = io.Copy(file, data)
+	if err != nil {
+		_ = os.Remove(path)
+	}
+	return err
+}
+
+// Transform generates a thumbnail and caches it.
+func (c *ThumbnailCache) Transform(key string, contentType string, data io.Reader) error {
+	if key == "" {
+		return NoKeySpecified
+	}
+	outputImage, err := Transform(c.ThumbnailerURL, contentType, data)
+	if err != nil {
+		return err
+	}
+	return c.SetThumbnail(key, outputImage)
+}

lib/thumbnailer/errors.go

+package thumbnailer
+
+type thumbnailerError struct {
+	Err string
+}
+
+// Error implements error.
+func (e *thumbnailerError) Error() string {
+	return e.Err
+}
+
+// NoCachedCopy means there is no cached copy for the specified key available.
+var NoCachedCopy error = &thumbnailerError{"no cached copy of the thumbnail requested is available"}
+
+// InputTooLarge means that the pixel size of the input image is too big to be thumbnailed.
+var InputTooLarge error = &thumbnailerError{"the input size in pixels is too large"}
+
+// NoKeySpecified means that no key was specified.
+var NoKeySpecified error = &thumbnailerError{"no key specified"}