Skip to content
GitLab
Explore
Sign in
Primary navigation
Search or go to…
Project
C
cdn-origin
Manage
Activity
Members
Labels
Plan
Issues
Issue boards
Milestones
Wiki
Code
Merge requests
Repository
Branches
Commits
Tags
Repository graph
Compare revisions
Snippets
Build
Pipelines
Jobs
Pipeline schedules
Artifacts
Deploy
Releases
Package Registry
Container Registry
Model registry
Operate
Environments
Terraform modules
Monitor
Incidents
Service Desk
Analyze
Value stream analytics
Contributor analytics
CI/CD analytics
Repository analytics
Model experiments
Help
Help
Support
GitLab documentation
Compare GitLab plans
Community forum
Contribute to GitLab
Provide feedback
Keyboard shortcuts
?
Snippets
Groups
Projects
Show more breadcrumbs
easrng
cdn-origin
Commits
021c8336
Commit
021c8336
authored
1 year ago
by
Dean
Browse files
Options
Downloads
Patches
Plain Diff
fix: copy thumbnail data to avoid buffer race
parent
63f1e246
No related branches found
No related tags found
No related merge requests found
Changes
4
Hide whitespace changes
Inline
Side-by-side
Showing
4 changed files
lib/thumbnailer/cache.go
+13
-7
13 additions, 7 deletions
lib/thumbnailer/cache.go
lib/thumbnailer/errors.go
+3
-0
3 additions, 0 deletions
lib/thumbnailer/errors.go
lib/thumbnailer/transform.go
+18
-9
18 additions, 9 deletions
lib/thumbnailer/transform.go
objects.sql
+17
-15
17 additions, 15 deletions
objects.sql
with
51 additions
and
31 deletions
lib/thumbnailer/cache.go
+
13
−
7
View file @
021c8336
...
@@ -25,6 +25,9 @@ func NewThumbnailCache(directory, thumbnailerURL string) *ThumbnailCache {
...
@@ -25,6 +25,9 @@ func NewThumbnailCache(directory, thumbnailerURL string) *ThumbnailCache {
// GetThumbnail returns a thumbnail that is cached. If no cached copy exists, a
// GetThumbnail returns a thumbnail that is cached. If no cached copy exists, a
// exists, a NoCachedCopy error is returned.
// exists, a NoCachedCopy error is returned.
func
(
c
*
ThumbnailCache
)
GetThumbnail
(
key
string
)
(
io
.
ReadCloser
,
error
)
{
func
(
c
*
ThumbnailCache
)
GetThumbnail
(
key
string
)
(
io
.
ReadCloser
,
error
)
{
if
key
==
""
{
return
nil
,
NoKeySpecified
}
path
:=
filepath
.
Join
(
c
.
Directory
,
key
)
path
:=
filepath
.
Join
(
c
.
Directory
,
key
)
data
,
err
:=
os
.
Open
(
path
)
data
,
err
:=
os
.
Open
(
path
)
if
os
.
IsNotExist
(
err
)
{
if
os
.
IsNotExist
(
err
)
{
...
@@ -35,27 +38,30 @@ func (c *ThumbnailCache) GetThumbnail(key string) (io.ReadCloser, error) {
...
@@ -35,27 +38,30 @@ func (c *ThumbnailCache) GetThumbnail(key string) (io.ReadCloser, error) {
// SetThumbnail stores a thumbnail with the specified key.
// SetThumbnail stores a thumbnail with the specified key.
func
(
c
*
ThumbnailCache
)
SetThumbnail
(
key
string
,
data
io
.
Reader
)
error
{
func
(
c
*
ThumbnailCache
)
SetThumbnail
(
key
string
,
data
io
.
Reader
)
error
{
if
key
==
""
{
return
NoKeySpecified
}
path
:=
filepath
.
Join
(
c
.
Directory
,
key
)
path
:=
filepath
.
Join
(
c
.
Directory
,
key
)
file
,
err
:=
os
.
Create
(
path
)
file
,
err
:=
os
.
Create
(
path
)
defer
file
.
Close
()
if
err
!=
nil
{
if
err
!=
nil
{
return
err
return
err
}
}
defer
file
.
Close
()
_
,
err
=
io
.
Copy
(
file
,
data
)
_
,
err
=
io
.
Copy
(
file
,
data
)
if
err
!=
nil
{
_
=
os
.
Remove
(
path
)
}
return
err
return
err
}
}
// Transform generates a thumbnail and caches it.
// Transform generates a thumbnail and caches it.
func
(
c
*
ThumbnailCache
)
Transform
(
key
string
,
contentType
string
,
data
io
.
Reader
)
error
{
func
(
c
*
ThumbnailCache
)
Transform
(
key
string
,
contentType
string
,
data
io
.
Reader
)
error
{
if
key
==
""
{
return
NoKeySpecified
}
outputImage
,
err
:=
Transform
(
c
.
ThumbnailerURL
,
contentType
,
data
)
outputImage
,
err
:=
Transform
(
c
.
ThumbnailerURL
,
contentType
,
data
)
if
err
!=
nil
{
if
err
!=
nil
{
return
err
return
err
}
}
return
c
.
SetThumbnail
(
key
,
outputImage
)
return
c
.
SetThumbnail
(
key
,
outputImage
)
}
}
// DeleteThumbnail deletes a thumbnail from the cache.
func
(
c
*
ThumbnailCache
)
DeleteThumbnail
(
key
string
)
error
{
path
:=
filepath
.
Join
(
c
.
Directory
,
key
)
return
os
.
Remove
(
path
)
}
This diff is collapsed.
Click to expand it.
lib/thumbnailer/errors.go
+
3
−
0
View file @
021c8336
...
@@ -14,3 +14,6 @@ var NoCachedCopy error = &thumbnailerError{"no cached copy of the thumbnail requ
...
@@ -14,3 +14,6 @@ var NoCachedCopy error = &thumbnailerError{"no cached copy of the thumbnail requ
// InputTooLarge means that the pixel size of the input image is too big to be thumbnailed.
// InputTooLarge means that the pixel size of the input image is too big to be thumbnailed.
var
InputTooLarge
error
=
&
thumbnailerError
{
"the input size in pixels is too large"
}
var
InputTooLarge
error
=
&
thumbnailerError
{
"the input size in pixels is too large"
}
// NoKeySpecified means that no key was specified.
var
NoKeySpecified
error
=
&
thumbnailerError
{
"no key specified"
}
This diff is collapsed.
Click to expand it.
lib/thumbnailer/transform.go
+
18
−
9
View file @
021c8336
...
@@ -11,10 +11,10 @@ import (
...
@@ -11,10 +11,10 @@ import (
// Accepted MIME types for thumbnails in map for easy checking
// Accepted MIME types for thumbnails in map for easy checking
var
thumbnailMIMETypes
=
map
[
string
]
struct
{}{
var
thumbnailMIMETypes
=
map
[
string
]
struct
{}{
"image/gif"
:
struct
{}
{},
"image/gif"
:
{},
"image/jpeg"
:
struct
{}
{},
"image/jpeg"
:
{},
"image/png"
:
struct
{}
{},
"image/png"
:
{},
"image/webp"
:
struct
{}
{},
"image/webp"
:
{},
}
}
// AcceptedMIMEType checks if a MIME type is suitable for thumbnailing.
// AcceptedMIMEType checks if a MIME type is suitable for thumbnailing.
...
@@ -27,15 +27,19 @@ func AcceptedMIMEType(mime string) bool {
...
@@ -27,15 +27,19 @@ func AcceptedMIMEType(mime string) bool {
// Transform takes an image io.Reader and sends it to the thumbnailer service
// Transform takes an image io.Reader and sends it to the thumbnailer service
// to be transcoded into a thumbnail.
// to be transcoded into a thumbnail.
func
Transform
(
thumbnailerURL
,
contentType
string
,
data
io
.
Reader
)
(
*
bytes
.
Buffer
,
error
)
{
func
Transform
(
thumbnailerURL
,
contentType
string
,
data
io
.
Reader
)
(
*
bytes
.
Buffer
,
error
)
{
// Set request and response
if
!
AcceptedMIMEType
(
contentType
)
{
return
nil
,
errors
.
Errorf
(
"invalid MIME type: %s"
,
contentType
)
}
req
:=
fasthttp
.
AcquireRequest
()
req
:=
fasthttp
.
AcquireRequest
()
res
:=
fasthttp
.
AcquireResponse
()
res
:=
fasthttp
.
AcquireResponse
()
defer
func
()
{
defer
func
()
{
fasthttp
.
ReleaseRequest
(
req
)
fasthttp
.
ReleaseRequest
(
req
)
fasthttp
.
ReleaseResponse
(
res
)
fasthttp
.
ReleaseResponse
(
res
)
}()
}()
req
.
Reset
()
req
.
Reset
()
res
.
Reset
()
req
.
Header
.
SetMethod
(
"POST"
)
req
.
Header
.
SetMethod
(
"POST"
)
req
.
SetRequestURI
(
thumbnailerURL
)
req
.
SetRequestURI
(
thumbnailerURL
)
req
.
Header
.
Set
(
"Content-Type"
,
contentType
)
req
.
Header
.
Set
(
"Content-Type"
,
contentType
)
...
@@ -43,9 +47,7 @@ func Transform(thumbnailerURL, contentType string, data io.Reader) (*bytes.Buffe
...
@@ -43,9 +47,7 @@ func Transform(thumbnailerURL, contentType string, data io.Reader) (*bytes.Buffe
if
err
!=
nil
{
if
err
!=
nil
{
return
nil
,
errors
.
Wrap
(
err
,
"failed to copy data to request"
)
return
nil
,
errors
.
Wrap
(
err
,
"failed to copy data to request"
)
}
}
res
.
Reset
()
// Do request
err
=
fasthttp
.
Do
(
req
,
res
)
err
=
fasthttp
.
Do
(
req
,
res
)
if
err
!=
nil
{
if
err
!=
nil
{
return
nil
,
errors
.
Wrap
(
err
,
"failed to make request to thumbnailer service"
)
return
nil
,
errors
.
Wrap
(
err
,
"failed to make request to thumbnailer service"
)
...
@@ -54,5 +56,12 @@ func Transform(thumbnailerURL, contentType string, data io.Reader) (*bytes.Buffe
...
@@ -54,5 +56,12 @@ func Transform(thumbnailerURL, contentType string, data io.Reader) (*bytes.Buffe
return
nil
,
errors
.
Errorf
(
"thumbnailer service failed to create thumbnail: %s"
,
string
(
res
.
Body
()))
return
nil
,
errors
.
Errorf
(
"thumbnailer service failed to create thumbnail: %s"
,
string
(
res
.
Body
()))
}
}
return
bytes
.
NewBuffer
(
res
.
Body
()),
nil
// Copy the response body to a buffer so we can return it safely.
// ReleaseResponse will return the buffer to the pool.
buf
:=
bytes
.
NewBuffer
(
nil
)
err
=
res
.
BodyWriteTo
(
buf
)
if
err
!=
nil
{
return
nil
,
errors
.
Wrap
(
err
,
"failed to copy response body to buffer"
)
}
return
buf
,
nil
}
}
This diff is collapsed.
Click to expand it.
objects.sql
+
17
−
15
View file @
021c8336
-- "objects" table schema
-- "objects" table schema
CREATE
TABLE
IF
NOT
EXISTS
objects
(
CREATE
TABLE
objects
(
bucket_key
VARCHAR
(
1088
)
NOT
NULL
UNIQUE
,
-- bucket + key (unique)
bucket_key
character
varying
(
1088
)
NOT
NULL
,
-- ${bucket}/${key} (unique)
bucket
VARCHAR
(
20
)
NOT
NULL
,
-- uint64 bucket ID ("public" for public bucket)
bucket
character
varying
(
20
)
NOT
NULL
,
-- Bucket ID ("public" for public bucket)
"key"
VARCHAR
(
1024
)
NOT
NULL
,
-- Full bucket path to file (including directory)
key
character
varying
(
1024
)
NOT
NULL
,
-- Full bucket path to file (including directory)
random_key
VARCHAR
(
1024
),
-- random key if used
dir
character
varying
(
1024
)
NOT
NULL
,
-- Directory of file (with trailing slash)
dir
VARCHAR
(
1024
)
NOT
NULL
,
-- Directory of file (with trailing slash)
type
integer
DEFAULT
0
NOT
NULL
,
-- Object type enumerable (0 = file, 1 = redirect)
"type"
integer
NOT
NULL
DEFAULT
0
,
-- Object type enumerable (0 = file, 1 = redirect)
backend_file_id
character
varying
(
33
)
DEFAULT
NULL
,
-- ID of file in backend storage
dest_url
VARCHAR
(
1024
)
DEFAULT
NULL
,
-- Destination URL for redirect object (only when object.type == 1)
dest_url
character
varying
(
4096
)
DEFAULT
NULL
,
-- Destination URL for redirect object (only when object.type == 1)
content_type
VARCHAR
(
255
)
DEFAULT
'application/octet-stream'
,
-- Content-Type of file
content_type
character
varying
(
255
)
DEFAULT
'application/octet-stream'
,
-- Content-Type of file
content_length
INT
DEFAULT
NULL
,
-- Content-Length of file
content_length
integer
,
-- Content-Length of file
associated_user
VARCHAR
(
36
)
DEFAULT
NULL
,
-- ID of user who uploaded file
created_at
timestamp
without
time
zone
DEFAULT
now
()
NOT
NULL
,
-- File creation timestamp
created_at
TIMESTAMP
NOT
NULL
DEFAULT
CURRENT_TIMESTAMP
,
-- File creation timestamp
random_key
character
varying
(
1024
)
DEFAULT
NULL
,
-- Random key if used
deleted_at
TIMESTAMP
DEFAULT
NULL
,
-- Deletion timestamp
associated_user
character
varying
(
36
)
DEFAULT
NULL
,
-- ID of user who uploaded file
delete_reason
VARCHAR
(
256
)
DEFAULT
NULL
,
-- Deletion reason
deleted_at
timestamp
without
time
zone
,
-- Deletion timestamp
md5_hash
VARCHAR
(
32
)
DEFAULT
NULL
-- MD5 hash of file contents (or destination URL)
delete_reason
character
varying
(
256
)
DEFAULT
NULL
::
character
varying
,
-- Deletion reason
sha256_hash
bytea
,
-- SHA256 hash of file contents (or destination URL)
md5_hash
bytea
-- MD5 hash of file contents (or destination URL)
);
);
-- Test file object: /index.md
-- Test file object: /index.md
...
...
This diff is collapsed.
Click to expand it.
Preview
0%
Loading
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Save comment
Cancel
Please
register
or
sign in
to comment