Dockerfile

-FROM golang:alpine
+FROM golang:1.21-alpine
 
 COPY go.mod /git/owo.codes/whats-this/api/
 COPY go.sum /git/owo.codes/whats-this/api/
@@ -7,9 +7,9 @@ COPY lib /git/owo.codes/whats-this/api/lib
 
 RUN apk add --no-cache --virtual .build-deps git build-base && \
     cd /git/owo.codes/whats-this/api && \
+    go mod download && \
     go build main.go && \
     apk del .build-deps
 
 WORKDIR /git/owo.codes/whats-this/api
 ENTRYPOINT ["./main"]
-

go.mod

 module owo.codes/whats-this/api
 
+go 1.21
+
+require (
+	github.com/akamensky/base58 v0.0.0-20210829145138-ce8bf8802e8f
+	github.com/go-chi/chi/v5 v5.0.10
+	github.com/go-chi/render v1.0.3
+	github.com/go-redis/redis v6.15.9+incompatible
+	github.com/gofrs/uuid v4.4.0+incompatible
+	github.com/lib/pq v1.10.9
+	github.com/pkg/errors v0.9.1
+	github.com/rs/zerolog v1.31.0
+	github.com/spf13/pflag v1.0.5
+	github.com/spf13/viper v1.17.0
+)
+
 require (
-	github.com/BurntSushi/toml v0.3.1 // indirect
-	github.com/akamensky/base58 v0.0.0-20170920141933-92b0f56f531a
-	github.com/go-chi/chi v4.0.1+incompatible
-	github.com/go-chi/render v1.0.1
-	github.com/go-redis/redis v6.15.1+incompatible
-	github.com/gofrs/uuid v3.2.0+incompatible
-	github.com/kr/pretty v0.1.0 // indirect
-	github.com/lib/pq v1.0.0
-	github.com/onsi/ginkgo v1.7.0 // indirect
-	github.com/onsi/gomega v1.4.3 // indirect
-	github.com/pkg/errors v0.8.1
-	github.com/rs/zerolog v1.11.0
-	github.com/spf13/pflag v1.0.3
-	github.com/spf13/viper v1.3.1
-	github.com/stretchr/testify v1.3.0 // indirect
-	golang.org/x/net v0.0.0-20190420063019-afa5a82059c6 // indirect
-	golang.org/x/sys v0.0.0-20190419153524-e8e3143a4f4a // indirect
-	gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect
+	github.com/ajg/form v1.5.1 // indirect
+	github.com/fsnotify/fsnotify v1.7.0 // indirect
+	github.com/hashicorp/hcl v1.0.0 // indirect
+	github.com/magiconair/properties v1.8.7 // indirect
+	github.com/mattn/go-colorable v0.1.13 // indirect
+	github.com/mattn/go-isatty v0.0.20 // indirect
+	github.com/mitchellh/mapstructure v1.5.0 // indirect
+	github.com/onsi/ginkgo v1.16.5 // indirect
+	github.com/onsi/gomega v1.18.1 // indirect
+	github.com/pelletier/go-toml/v2 v2.1.0 // indirect
+	github.com/sagikazarmark/locafero v0.3.0 // indirect
+	github.com/sagikazarmark/slog-shim v0.1.0 // indirect
+	github.com/sourcegraph/conc v0.3.0 // indirect
+	github.com/spf13/afero v1.10.0 // indirect
+	github.com/spf13/cast v1.5.1 // indirect
+	github.com/spf13/jwalterweatherman v1.1.0 // indirect
+	github.com/subosito/gotenv v1.6.0 // indirect
+	go.uber.org/atomic v1.11.0 // indirect
+	go.uber.org/multierr v1.11.0 // indirect
+	golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa // indirect
+	golang.org/x/sys v0.14.0 // indirect
+	golang.org/x/text v0.14.0 // indirect
+	gopkg.in/ini.v1 v1.67.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

lib/apierrors/errors.go

@@ -61,6 +61,9 @@ var (
 
 	// FileIsNotBanned is a 404 not found error.
 	FileIsNotBanned = APIError{false, 404, "specified file is not banned", false}
+
+	// InvalidObjectFilter is a 400 bad request error.
+	InvalidObjectFilter = APIError{false, 400, `invalid filter, must be "", "files" or "links"`, false}
 )
 
 // Pomf errors

lib/db/queries.go

@@ -217,13 +217,19 @@ func scanObject(scanner scanner) (Object, error) {
 
 // ListObjectsByAssociatedUser returns all objects (paginated) associated with a
 // user.
-func ListObjectsByAssociatedUser(userID string, asc bool, offset, limit int) ([]Object, error) {
+func ListObjectsByAssociatedUser(userID string, typ int, asc bool, offset, limit int) ([]Object, error) {
 	objects := []Object{}
 	order := "DESC"
 	if asc {
 		order = "ASC"
 	}
-	rows, err := DB.Query(fmt.Sprintf(listObjectsByAssociatedUser, order), userID, limit, offset)
+
+	typeFilter := "!= 2"
+	if typ != -1 {
+		typeFilter = fmt.Sprintf("= %v", typ)
+	}
+
+	rows, err := DB.Query(fmt.Sprintf(listObjectsByAssociatedUser, typeFilter, order), userID, limit, offset)
 	if err != nil {
 		return objects, err
 	}
@@ -240,12 +246,38 @@ func ListObjectsByAssociatedUser(userID string, asc bool, offset, limit int) ([]
 	return objects, nil
 }
 
+// CountObjectsByAssociatedUser returns the count of all objects associated with a user.
+func CountObjectsByAssociatedUser(userID string, typ int) (int, error) {
+	typeFilter := "!= 2"
+	if typ != -1 {
+		typeFilter = fmt.Sprintf("= %v", typ)
+	}
+
+	var count int
+	query := fmt.Sprintf(countObjectsByAssociatedUser, typeFilter)
+	err := DB.QueryRow(query, userID).Scan(&count)
+	if err != nil {
+		return 0, err
+	}
+	return count, nil
+}
+
 // GetObject returns an object.
 func GetObject(bucket, key string) (Object, error) {
 	row := DB.QueryRow(getObjectByBucketKey, fmt.Sprintf("%s/%s", bucket, key))
 	return scanObject(row)
 }
 
+// CheckIfObjectExists returns true if an object exists with the specified hash.
+func CheckIfObjectExists(sha256 []byte) (bool, error) {
+	var count int
+	err := DB.QueryRow(countOfObjectsBySHA256, sha256).Scan(&count)
+	if err != nil {
+		return false, err
+	}
+	return count > 0, nil
+}
+
 // UpdateObjectToTombstoneByBucketKey sets an object to be a tombstone by bucket and key.
 func UpdateObjectToTombstoneByBucketKey(bucket, key string, reason *string, retainAssociatedUser bool, retainHashes bool) error {
 	sql := updateObjectToTombstoneByBucketKey

lib/db/sql.go

@@ -65,6 +65,16 @@ VALUES
 	($1, $2)
 `
 
+var countObjectsByAssociatedUser = `
+SELECT
+	COUNT(*) as count
+FROM
+	objects
+WHERE
+	associated_user = $1 AND
+	"type" %v
+`
+
 var listObjectsByAssociatedUser = `
 SELECT
 	bucket,
@@ -84,7 +94,7 @@ FROM
 	objects
 WHERE
 	associated_user = $1 AND
-	"type" != 2
+	"type" %v
 ORDER BY
 	created_at %s
 LIMIT $2
@@ -113,27 +123,14 @@ WHERE
 LIMIT 1
 `
 
-var getObjectBySHA256Hash = `
+var countOfObjectsBySHA256 = `
 SELECT
-	bucket,
-	key,
-	dir,
-	"type",
-	dest_url,
-	content_type,
-	content_length,
-	created_at,
-	deleted_at,
-	delete_reason,
-	md5_hash,
-	sha256_hash,
-	associated_user
+	COUNT(*)
 FROM
 	objects
 WHERE
-	"type" = 0,
+	"type" = 0 AND
 	sha256_hash = $1
-LIMIT 1
 `
 
 var updateObjectToTombstoneByBucketKey = `

lib/routes/deleteobject.go

@@ -90,12 +90,20 @@ func DeleteObject(w http.ResponseWriter, r *http.Request) {
 
 	// Delete file from disk if type was 0
 	if object.Type == 0 {
-		destPath := filepath.Join(viper.GetString("files.storageLocation"), *object.SHA256Hash)
-		err = os.Remove(destPath)
+		exists, err := db.CheckIfObjectExists(object.SHA256HashBytes)
 		if err != nil {
-			log.Error().Err(err).Msg("failed to delete object file from disk")
+			log.Error().Err(err).Str("hash", *object.SHA256Hash).Msg("failed to check if file exists")
 			panic(apierrors.InternalServerError)
 		}
+
+		if !exists {
+			destPath := filepath.Join(viper.GetString("files.storageLocation"), *object.SHA256Hash)
+			err = os.Remove(destPath)
+			if err != nil {
+				log.Error().Err(err).Msg("failed to delete object file from disk")
+				panic(apierrors.InternalServerError)
+			}
+		}
 	}
 
 	// Return tombstone response

lib/routes/listobjects.go

@@ -14,13 +14,20 @@ import (
 	"github.com/spf13/viper"
 )
 
-// Maximum objects per page
-const maxLimit = 100
+const (
+	// Maximum objects per page
+	maxLimit = 100
+
+	// filter keys for file vs short link.
+	filterFiles = "file"
+	filterLinks = "link"
+)
 
 // listObjectsResponse is the response format for ListObjects.
 type listObjectsResponse struct {
-	Success bool        `json:"success"`
-	Data    []db.Object `json:"data"`
+	Success      bool        `json:"success"`
+	TotalObjects int         `json:"total_objects"`
+	Data         []db.Object `json:"data"`
 }
 
 // ListObjects returns a paginated list of all objects owned by a user.
@@ -41,12 +48,17 @@ func ListObjects(w http.ResponseWriter, r *http.Request) {
 		panic(apierrors.InternalServerError)
 	}
 
-	// Determine offset and limit information
-	query := r.URL.Query()
-	l := query.Get("limit")
-	limit, err := strconv.Atoi(l)
-	if err != nil || limit < 0 {
-		panic(apierrors.InvalidOffsetOrLimit)
+	// Determine offset, limit and filter params
+	var (
+		query = r.URL.Query()
+		limit = maxLimit
+		l     = query.Get("limit")
+	)
+	if l != "" {
+		limit, err = strconv.Atoi(l)
+		if err != nil || limit < 0 {
+			panic(apierrors.InvalidOffsetOrLimit)
+		}
 	}
 	if limit > maxLimit {
 		panic(apierrors.LimitTooLarge)
@@ -60,9 +72,26 @@ func ListObjects(w http.ResponseWriter, r *http.Request) {
 	if query.Get("order") == "asc" {
 		asc = true
 	}
+	filter := query.Get("type")
+	if filter != "" && filter != filterFiles && filter != filterLinks {
+		panic(apierrors.InvalidObjectFilter)
+	}
+
+	f := -1
+	if filter == filterFiles {
+		f = 0
+	} else if filter == filterLinks {
+		f = 1
+	}
 
 	// Get the data
-	objects, err := db.ListObjectsByAssociatedUser(user.ID, asc, offset, limit)
+	count, err := db.CountObjectsByAssociatedUser(user.ID, f)
+	if err != nil {
+		log.Error().Err(err).Msg("failed to count objects for user")
+		panic(apierrors.InternalServerError)
+	}
+
+	objects, err := db.ListObjectsByAssociatedUser(user.ID, f, asc, offset, limit)
 	if err != nil {
 		log.Error().Err(err).Msg("failed to list objects for user")
 		panic(apierrors.InternalServerError)
@@ -75,5 +104,5 @@ func ListObjects(w http.ResponseWriter, r *http.Request) {
 	// Return response
 	w.Header().Set("Content-Type", "application/json")
 	w.WriteHeader(http.StatusOK)
-	render.JSON(w, r, listObjectsResponse{true, objects})
+	render.JSON(w, r, listObjectsResponse{true, count, objects})
 }

lib/routes/uploadpomf.go

@@ -357,7 +357,11 @@ func UploadPomf(associateObjectsWithUser bool, simpleResponse bool) func(http.Re
 				if fResponse.Success {
 					w.Write([]byte(fmt.Sprintf("%s,200\n", fResponse.URL)))
 				} else {
-					w.Write([]byte(",500\n"))
+					status := fResponse.StatusCode
+					if status == 0 {
+						status = 500
+					}
+					w.Write([]byte(fmt.Sprintf(",%v\n", status)))
 				}
 			}
 		} else {

main.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"io"
+	"net"
 	"net/http"
 	"os"
 	"os/signal"
@@ -15,7 +16,7 @@ import (
 	"owo.codes/whats-this/api/lib/ratelimiter"
 	"owo.codes/whats-this/api/lib/routes"
 
-	"github.com/go-chi/chi"
+	chi "github.com/go-chi/chi/v5"
 	"github.com/rs/zerolog"
 	"github.com/rs/zerolog/log"
 	"github.com/spf13/pflag"
@@ -198,8 +199,9 @@ func main() {
 	// Create HTTP server on specified listening address
 	listenAddress := viper.GetString("http.listenAddress")
 	server := http.Server{
-		Addr:    listenAddress,
-		Handler: chi.ServerBaseContext(baseCtx, r),
+		Addr:        listenAddress,
+		Handler:     r,
+		BaseContext: func(_ net.Listener) context.Context { return baseCtx },
 	}
 
 	// Listen for interrupts (^C) and exit gracefully

sql/README

+This directory contains the PostgreSQL scripts required to prepare the database
+for use with the API server and cdn-origin.
+
+Reference:
+- v1.sql: the raw, unprocessed output of pg_dump --no-owner --no-privileges
+  --schema-only over the main owo database, version 1.

sql/v1.sql

+--
+-- PostgreSQL database dump
+--
+
+-- Dumped from database version 14.5
+-- Dumped by pg_dump version 14.5
+
+SET statement_timeout = 0;
+SET lock_timeout = 0;
+SET idle_in_transaction_session_timeout = 0;
+SET client_encoding = 'UTF8';
+SET standard_conforming_strings = on;
+SELECT pg_catalog.set_config('search_path', '', false);
+SET check_function_bodies = false;
+SET xmloption = content;
+SET client_min_messages = warning;
+SET row_security = off;
+
+SET default_tablespace = '';
+
+SET default_table_access_method = heap;
+
+--
+-- Name: file_bans; Type: TABLE; Schema: public; Owner: -
+--
+
+CREATE TABLE public.file_bans (
+    sha256_hash bytea NOT NULL,
+    did_quarantine boolean DEFAULT false NOT NULL,
+    reason integer DEFAULT 0 NOT NULL,
+    description character varying(1024) DEFAULT NULL::character varying,
+    malware_name character varying(256) DEFAULT NULL::character varying
+);
+
+
+--
+-- Name: objects; Type: TABLE; Schema: public; Owner: -
+--
+
+CREATE TABLE public.objects (
+    bucket_key character varying(1088) NOT NULL,
+    bucket character varying(20) NOT NULL,
+    key character varying(1024) NOT NULL,
+    dir character varying(1024) NOT NULL,
+    type integer DEFAULT 0 NOT NULL,
+    backend_file_id character varying(33) DEFAULT NULL::character varying,
+    dest_url character varying(4096) DEFAULT NULL::character varying,
+    content_type character varying(255) DEFAULT 'application/octet-stream'::character varying,
+    content_length integer,
+    created_at timestamp without time zone DEFAULT now() NOT NULL,
+    random_key character varying(1024) DEFAULT NULL::character varying,
+    associated_user character varying(36) DEFAULT NULL::character varying,
+    deleted_at timestamp without time zone,
+    delete_reason character varying(256) DEFAULT NULL::character varying,
+    sha256_hash bytea,
+    md5_hash bytea
+);
+
+
+--
+-- Name: tokens; Type: TABLE; Schema: public; Owner: -
+--
+
+CREATE TABLE public.tokens (
+    id integer NOT NULL,
+    user_id character varying(255) NOT NULL,
+    token character varying(255) NOT NULL
+);
+
+
+--
+-- Name: tokens_id_seq; Type: SEQUENCE; Schema: public; Owner: -
+--
+
+CREATE SEQUENCE public.tokens_id_seq
+    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+
+--
+-- Name: tokens_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
+--
+
+ALTER SEQUENCE public.tokens_id_seq OWNED BY public.tokens.id;
+
+
+--
+-- Name: users; Type: TABLE; Schema: public; Owner: -
+--
+
+CREATE TABLE public.users (
+    id character varying(255) NOT NULL,
+    username character varying(255) NOT NULL,
+    email character varying(255) NOT NULL,
+    is_admin boolean DEFAULT false NOT NULL,
+    is_blocked boolean DEFAULT false NOT NULL,
+    username_lower character varying(255) NOT NULL,
+    bucket_capacity integer
+);
+
+
+--
+-- Name: tokens id; Type: DEFAULT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.tokens ALTER COLUMN id SET DEFAULT nextval('public.tokens_id_seq'::regclass);
+
+
+--
+-- Name: file_bans file_bans_sha256_hash_key; Type: CONSTRAINT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.file_bans
+    ADD CONSTRAINT file_bans_sha256_hash_key UNIQUE (sha256_hash);
+
+
+--
+-- Name: objects objects_bucket_key_key; Type: CONSTRAINT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.objects
+    ADD CONSTRAINT objects_bucket_key_key UNIQUE (bucket_key);
+
+
+--
+-- Name: tokens tokens_pk; Type: CONSTRAINT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.tokens
+    ADD CONSTRAINT tokens_pk PRIMARY KEY (id);
+
+
+--
+-- Name: tokens tokens_token_key; Type: CONSTRAINT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.tokens
+    ADD CONSTRAINT tokens_token_key UNIQUE (token);
+
+
+--
+-- Name: users username_lower_unique; Type: CONSTRAINT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.users
+    ADD CONSTRAINT username_lower_unique UNIQUE (username_lower);
+
+
+--
+-- Name: users users_email_key; Type: CONSTRAINT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.users
+    ADD CONSTRAINT users_email_key UNIQUE (email);
+
+
+--
+-- Name: users users_pk; Type: CONSTRAINT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.users
+    ADD CONSTRAINT users_pk PRIMARY KEY (id);
+
+
+--
+-- Name: users users_username_key; Type: CONSTRAINT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.users
+    ADD CONSTRAINT users_username_key UNIQUE (username);
+
+
+--
+-- Name: objects_associated_user; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE INDEX objects_associated_user ON public.objects USING btree (associated_user);
+
+
+--
+-- Name: objects_bucket_random_key; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE INDEX objects_bucket_random_key ON public.objects USING btree (bucket, random_key);
+
+
+--
+-- Name: objects_sha256_hash_idx; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE INDEX objects_sha256_hash_idx ON public.objects USING btree (sha256_hash);
+
+
+--
+-- PostgreSQL database dump complete
+--
+