Skip to content
Snippets Groups Projects
Commit 91ec1c6c authored by Nicolas George's avatar Nicolas George
Browse files

loco: take decode overflow into account.

Commit 2bf09826 introduced an overflow check in loco_decode_plane,
but the error code is never taken into account, leading to
completely idiotic return values.
parent b2814b03
No related branches found
No related tags found
No related merge requests found
...@@ -179,51 +179,56 @@ static int decode_frame(AVCodecContext *avctx, ...@@ -179,51 +179,56 @@ static int decode_frame(AVCodecContext *avctx,
} }
p->key_frame = 1; p->key_frame = 1;
#define ADVANCE_BY_DECODED do { \
if (decoded < 0) goto stop; \
buf += decoded; buf_size -= decoded; \
} while(0)
switch(l->mode) { switch(l->mode) {
case LOCO_CYUY2: case LOCO_YUY2: case LOCO_UYVY: case LOCO_CYUY2: case LOCO_YUY2: case LOCO_UYVY:
decoded = loco_decode_plane(l, p->data[0], avctx->width, avctx->height, decoded = loco_decode_plane(l, p->data[0], avctx->width, avctx->height,
p->linesize[0], buf, buf_size, 1); p->linesize[0], buf, buf_size, 1);
buf += decoded; buf_size -= decoded; ADVANCE_BY_DECODED;
decoded = loco_decode_plane(l, p->data[1], avctx->width / 2, avctx->height, decoded = loco_decode_plane(l, p->data[1], avctx->width / 2, avctx->height,
p->linesize[1], buf, buf_size, 1); p->linesize[1], buf, buf_size, 1);
buf += decoded; buf_size -= decoded; ADVANCE_BY_DECODED;
decoded = loco_decode_plane(l, p->data[2], avctx->width / 2, avctx->height, decoded = loco_decode_plane(l, p->data[2], avctx->width / 2, avctx->height,
p->linesize[2], buf, buf_size, 1); p->linesize[2], buf, buf_size, 1);
break; break;
case LOCO_CYV12: case LOCO_YV12: case LOCO_CYV12: case LOCO_YV12:
decoded = loco_decode_plane(l, p->data[0], avctx->width, avctx->height, decoded = loco_decode_plane(l, p->data[0], avctx->width, avctx->height,
p->linesize[0], buf, buf_size, 1); p->linesize[0], buf, buf_size, 1);
buf += decoded; buf_size -= decoded; ADVANCE_BY_DECODED;
decoded = loco_decode_plane(l, p->data[2], avctx->width / 2, avctx->height / 2, decoded = loco_decode_plane(l, p->data[2], avctx->width / 2, avctx->height / 2,
p->linesize[2], buf, buf_size, 1); p->linesize[2], buf, buf_size, 1);
buf += decoded; buf_size -= decoded; ADVANCE_BY_DECODED;
decoded = loco_decode_plane(l, p->data[1], avctx->width / 2, avctx->height / 2, decoded = loco_decode_plane(l, p->data[1], avctx->width / 2, avctx->height / 2,
p->linesize[1], buf, buf_size, 1); p->linesize[1], buf, buf_size, 1);
break; break;
case LOCO_CRGB: case LOCO_RGB: case LOCO_CRGB: case LOCO_RGB:
decoded = loco_decode_plane(l, p->data[0] + p->linesize[0]*(avctx->height-1), avctx->width, avctx->height, decoded = loco_decode_plane(l, p->data[0] + p->linesize[0]*(avctx->height-1), avctx->width, avctx->height,
-p->linesize[0], buf, buf_size, 3); -p->linesize[0], buf, buf_size, 3);
buf += decoded; buf_size -= decoded; ADVANCE_BY_DECODED;
decoded = loco_decode_plane(l, p->data[0] + p->linesize[0]*(avctx->height-1) + 1, avctx->width, avctx->height, decoded = loco_decode_plane(l, p->data[0] + p->linesize[0]*(avctx->height-1) + 1, avctx->width, avctx->height,
-p->linesize[0], buf, buf_size, 3); -p->linesize[0], buf, buf_size, 3);
buf += decoded; buf_size -= decoded; ADVANCE_BY_DECODED;
decoded = loco_decode_plane(l, p->data[0] + p->linesize[0]*(avctx->height-1) + 2, avctx->width, avctx->height, decoded = loco_decode_plane(l, p->data[0] + p->linesize[0]*(avctx->height-1) + 2, avctx->width, avctx->height,
-p->linesize[0], buf, buf_size, 3); -p->linesize[0], buf, buf_size, 3);
break; break;
case LOCO_CRGBA: case LOCO_RGBA: case LOCO_CRGBA: case LOCO_RGBA:
decoded = loco_decode_plane(l, p->data[0], avctx->width, avctx->height, decoded = loco_decode_plane(l, p->data[0], avctx->width, avctx->height,
p->linesize[0], buf, buf_size, 4); p->linesize[0], buf, buf_size, 4);
buf += decoded; buf_size -= decoded; ADVANCE_BY_DECODED;
decoded = loco_decode_plane(l, p->data[0] + 1, avctx->width, avctx->height, decoded = loco_decode_plane(l, p->data[0] + 1, avctx->width, avctx->height,
p->linesize[0], buf, buf_size, 4); p->linesize[0], buf, buf_size, 4);
buf += decoded; buf_size -= decoded; ADVANCE_BY_DECODED;
decoded = loco_decode_plane(l, p->data[0] + 2, avctx->width, avctx->height, decoded = loco_decode_plane(l, p->data[0] + 2, avctx->width, avctx->height,
p->linesize[0], buf, buf_size, 4); p->linesize[0], buf, buf_size, 4);
buf += decoded; buf_size -= decoded; ADVANCE_BY_DECODED;
decoded = loco_decode_plane(l, p->data[0] + 3, avctx->width, avctx->height, decoded = loco_decode_plane(l, p->data[0] + 3, avctx->width, avctx->height,
p->linesize[0], buf, buf_size, 4); p->linesize[0], buf, buf_size, 4);
break; break;
} }
stop:
*data_size = sizeof(AVFrame); *data_size = sizeof(AVFrame);
*(AVFrame*)data = l->pic; *(AVFrame*)data = l->pic;
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment