blob: a756dab2111e51f6fb59d9e5124ddeec478f687a
1 | /* |
2 | * VMware Screen Codec (VMnc) decoder |
3 | * Copyright (c) 2006 Konstantin Shishkov |
4 | * |
5 | * This file is part of FFmpeg. |
6 | * |
7 | * FFmpeg is free software; you can redistribute it and/or |
8 | * modify it under the terms of the GNU Lesser General Public |
9 | * License as published by the Free Software Foundation; either |
10 | * version 2.1 of the License, or (at your option) any later version. |
11 | * |
12 | * FFmpeg is distributed in the hope that it will be useful, |
13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of |
14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU |
15 | * Lesser General Public License for more details. |
16 | * |
17 | * You should have received a copy of the GNU Lesser General Public |
18 | * License along with FFmpeg; if not, write to the Free Software |
19 | * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA |
20 | */ |
21 | |
22 | /** |
23 | * @file |
24 | * VMware Screen Codec (VMnc) decoder |
25 | * As Alex Beregszaszi discovered, this is effectively RFB data dump |
26 | */ |
27 | |
28 | #include <stdio.h> |
29 | #include <stdlib.h> |
30 | |
31 | #include "libavutil/common.h" |
32 | #include "libavutil/intreadwrite.h" |
33 | #include "avcodec.h" |
34 | #include "internal.h" |
35 | #include "bytestream.h" |
36 | |
37 | enum EncTypes { |
38 | MAGIC_WMVd = 0x574D5664, |
39 | MAGIC_WMVe, |
40 | MAGIC_WMVf, |
41 | MAGIC_WMVg, |
42 | MAGIC_WMVh, |
43 | MAGIC_WMVi, |
44 | MAGIC_WMVj |
45 | }; |
46 | |
47 | enum HexTile_Flags { |
48 | HT_RAW = 1, // tile is raw |
49 | HT_BKG = 2, // background color is present |
50 | HT_FG = 4, // foreground color is present |
51 | HT_SUB = 8, // subrects are present |
52 | HT_CLR = 16 // each subrect has own color |
53 | }; |
54 | |
55 | /* |
56 | * Decoder context |
57 | */ |
58 | typedef struct VmncContext { |
59 | AVCodecContext *avctx; |
60 | AVFrame *pic; |
61 | |
62 | int bpp; |
63 | int bpp2; |
64 | int bigendian; |
65 | uint8_t pal[768]; |
66 | int width, height; |
67 | GetByteContext gb; |
68 | |
69 | /* cursor data */ |
70 | int cur_w, cur_h; |
71 | int cur_x, cur_y; |
72 | int cur_hx, cur_hy; |
73 | uint8_t *curbits, *curmask; |
74 | uint8_t *screendta; |
75 | } VmncContext; |
76 | |
77 | /* read pixel value from stream */ |
78 | static av_always_inline int vmnc_get_pixel(GetByteContext *gb, int bpp, int be) |
79 | { |
80 | switch (bpp * 2 + be) { |
81 | case 2: |
82 | case 3: |
83 | return bytestream2_get_byte(gb); |
84 | case 4: |
85 | return bytestream2_get_le16(gb); |
86 | case 5: |
87 | return bytestream2_get_be16(gb); |
88 | case 8: |
89 | return bytestream2_get_le32(gb); |
90 | case 9: |
91 | return bytestream2_get_be32(gb); |
92 | default: return 0; |
93 | } |
94 | } |
95 | |
96 | static void load_cursor(VmncContext *c) |
97 | { |
98 | int i, j, p; |
99 | const int bpp = c->bpp2; |
100 | uint8_t *dst8 = c->curbits; |
101 | uint16_t *dst16 = (uint16_t *)c->curbits; |
102 | uint32_t *dst32 = (uint32_t *)c->curbits; |
103 | |
104 | for (j = 0; j < c->cur_h; j++) { |
105 | for (i = 0; i < c->cur_w; i++) { |
106 | p = vmnc_get_pixel(&c->gb, bpp, c->bigendian); |
107 | if (bpp == 1) |
108 | *dst8++ = p; |
109 | if (bpp == 2) |
110 | *dst16++ = p; |
111 | if (bpp == 4) |
112 | *dst32++ = p; |
113 | } |
114 | } |
115 | dst8 = c->curmask; |
116 | dst16 = (uint16_t*)c->curmask; |
117 | dst32 = (uint32_t*)c->curmask; |
118 | for (j = 0; j < c->cur_h; j++) { |
119 | for (i = 0; i < c->cur_w; i++) { |
120 | p = vmnc_get_pixel(&c->gb, bpp, c->bigendian); |
121 | if (bpp == 1) |
122 | *dst8++ = p; |
123 | if (bpp == 2) |
124 | *dst16++ = p; |
125 | if (bpp == 4) |
126 | *dst32++ = p; |
127 | } |
128 | } |
129 | } |
130 | |
131 | static void put_cursor(uint8_t *dst, int stride, VmncContext *c, int dx, int dy) |
132 | { |
133 | int i, j; |
134 | int w, h, x, y; |
135 | w = c->cur_w; |
136 | if (c->width < c->cur_x + c->cur_w) |
137 | w = c->width - c->cur_x; |
138 | h = c->cur_h; |
139 | if (c->height < c->cur_y + c->cur_h) |
140 | h = c->height - c->cur_y; |
141 | x = c->cur_x; |
142 | y = c->cur_y; |
143 | if (x < 0) { |
144 | w += x; |
145 | x = 0; |
146 | } |
147 | if (y < 0) { |
148 | h += y; |
149 | y = 0; |
150 | } |
151 | |
152 | if ((w < 1) || (h < 1)) |
153 | return; |
154 | dst += x * c->bpp2 + y * stride; |
155 | |
156 | if (c->bpp2 == 1) { |
157 | uint8_t *cd = c->curbits, *msk = c->curmask; |
158 | for (j = 0; j < h; j++) { |
159 | for (i = 0; i < w; i++) |
160 | dst[i] = (dst[i] & cd[i]) ^ msk[i]; |
161 | msk += c->cur_w; |
162 | cd += c->cur_w; |
163 | dst += stride; |
164 | } |
165 | } else if (c->bpp2 == 2) { |
166 | uint16_t *cd = (uint16_t*)c->curbits, *msk = (uint16_t*)c->curmask; |
167 | uint16_t *dst2; |
168 | for (j = 0; j < h; j++) { |
169 | dst2 = (uint16_t*)dst; |
170 | for (i = 0; i < w; i++) |
171 | dst2[i] = (dst2[i] & cd[i]) ^ msk[i]; |
172 | msk += c->cur_w; |
173 | cd += c->cur_w; |
174 | dst += stride; |
175 | } |
176 | } else if (c->bpp2 == 4) { |
177 | uint32_t *cd = (uint32_t*)c->curbits, *msk = (uint32_t*)c->curmask; |
178 | uint32_t *dst2; |
179 | for (j = 0; j < h; j++) { |
180 | dst2 = (uint32_t*)dst; |
181 | for (i = 0; i < w; i++) |
182 | dst2[i] = (dst2[i] & cd[i]) ^ msk[i]; |
183 | msk += c->cur_w; |
184 | cd += c->cur_w; |
185 | dst += stride; |
186 | } |
187 | } |
188 | } |
189 | |
190 | /* fill rectangle with given color */ |
191 | static av_always_inline void paint_rect(uint8_t *dst, int dx, int dy, |
192 | int w, int h, int color, |
193 | int bpp, int stride) |
194 | { |
195 | int i, j; |
196 | dst += dx * bpp + dy * stride; |
197 | if (bpp == 1) { |
198 | for (j = 0; j < h; j++) { |
199 | memset(dst, color, w); |
200 | dst += stride; |
201 | } |
202 | } else if (bpp == 2) { |
203 | uint16_t *dst2; |
204 | for (j = 0; j < h; j++) { |
205 | dst2 = (uint16_t*)dst; |
206 | for (i = 0; i < w; i++) |
207 | *dst2++ = color; |
208 | dst += stride; |
209 | } |
210 | } else if (bpp == 4) { |
211 | uint32_t *dst2; |
212 | for (j = 0; j < h; j++) { |
213 | dst2 = (uint32_t*)dst; |
214 | for (i = 0; i < w; i++) |
215 | dst2[i] = color; |
216 | dst += stride; |
217 | } |
218 | } |
219 | } |
220 | |
221 | static av_always_inline void paint_raw(uint8_t *dst, int w, int h, |
222 | GetByteContext *gb, int bpp, |
223 | int be, int stride) |
224 | { |
225 | int i, j, p; |
226 | for (j = 0; j < h; j++) { |
227 | for (i = 0; i < w; i++) { |
228 | p = vmnc_get_pixel(gb, bpp, be); |
229 | switch (bpp) { |
230 | case 1: |
231 | dst[i] = p; |
232 | break; |
233 | case 2: |
234 | ((uint16_t*)dst)[i] = p; |
235 | break; |
236 | case 4: |
237 | ((uint32_t*)dst)[i] = p; |
238 | break; |
239 | } |
240 | } |
241 | dst += stride; |
242 | } |
243 | } |
244 | |
245 | static int decode_hextile(VmncContext *c, uint8_t* dst, GetByteContext *gb, |
246 | int w, int h, int stride) |
247 | { |
248 | int i, j, k; |
249 | int bg = 0, fg = 0, rects, color, flags, xy, wh; |
250 | const int bpp = c->bpp2; |
251 | uint8_t *dst2; |
252 | int bw = 16, bh = 16; |
253 | |
254 | for (j = 0; j < h; j += 16) { |
255 | dst2 = dst; |
256 | bw = 16; |
257 | if (j + 16 > h) |
258 | bh = h - j; |
259 | for (i = 0; i < w; i += 16, dst2 += 16 * bpp) { |
260 | if (bytestream2_get_bytes_left(gb) <= 0) { |
261 | av_log(c->avctx, AV_LOG_ERROR, "Premature end of data!\n"); |
262 | return AVERROR_INVALIDDATA; |
263 | } |
264 | if (i + 16 > w) |
265 | bw = w - i; |
266 | flags = bytestream2_get_byte(gb); |
267 | if (flags & HT_RAW) { |
268 | if (bytestream2_get_bytes_left(gb) < bw * bh * bpp) { |
269 | av_log(c->avctx, AV_LOG_ERROR, "Premature end of data!\n"); |
270 | return AVERROR_INVALIDDATA; |
271 | } |
272 | paint_raw(dst2, bw, bh, gb, bpp, c->bigendian, stride); |
273 | } else { |
274 | if (flags & HT_BKG) |
275 | bg = vmnc_get_pixel(gb, bpp, c->bigendian); |
276 | if (flags & HT_FG) |
277 | fg = vmnc_get_pixel(gb, bpp, c->bigendian); |
278 | rects = 0; |
279 | if (flags & HT_SUB) |
280 | rects = bytestream2_get_byte(gb); |
281 | color = !!(flags & HT_CLR); |
282 | |
283 | paint_rect(dst2, 0, 0, bw, bh, bg, bpp, stride); |
284 | |
285 | if (bytestream2_get_bytes_left(gb) < rects * (color * bpp + 2)) { |
286 | av_log(c->avctx, AV_LOG_ERROR, "Premature end of data!\n"); |
287 | return AVERROR_INVALIDDATA; |
288 | } |
289 | for (k = 0; k < rects; k++) { |
290 | int rect_x, rect_y, rect_w, rect_h; |
291 | if (color) |
292 | fg = vmnc_get_pixel(gb, bpp, c->bigendian); |
293 | xy = bytestream2_get_byte(gb); |
294 | wh = bytestream2_get_byte(gb); |
295 | |
296 | rect_x = xy >> 4; |
297 | rect_y = xy & 0xF; |
298 | rect_w = (wh >> 4) + 1; |
299 | rect_h = (wh & 0xF) + 1; |
300 | |
301 | if (rect_x + rect_w > w - i || rect_y + rect_h > h - j) { |
302 | av_log(c->avctx, AV_LOG_ERROR, "Rectangle outside picture\n"); |
303 | return AVERROR_INVALIDDATA; |
304 | } |
305 | |
306 | paint_rect(dst2, rect_x, rect_y, |
307 | rect_w, rect_h, fg, bpp, stride); |
308 | } |
309 | } |
310 | } |
311 | dst += stride * 16; |
312 | } |
313 | return 0; |
314 | } |
315 | |
316 | static void reset_buffers(VmncContext *c) |
317 | { |
318 | av_freep(&c->curbits); |
319 | av_freep(&c->curmask); |
320 | av_freep(&c->screendta); |
321 | c->cur_w = c->cur_h = 0; |
322 | c->cur_hx = c->cur_hy = 0; |
323 | |
324 | } |
325 | |
326 | static int decode_frame(AVCodecContext *avctx, void *data, int *got_frame, |
327 | AVPacket *avpkt) |
328 | { |
329 | const uint8_t *buf = avpkt->data; |
330 | int buf_size = avpkt->size; |
331 | VmncContext * const c = avctx->priv_data; |
332 | GetByteContext *gb = &c->gb; |
333 | uint8_t *outptr; |
334 | int dx, dy, w, h, depth, enc, chunks, res, size_left, ret; |
335 | |
336 | if ((ret = ff_reget_buffer(avctx, c->pic)) < 0) |
337 | return ret; |
338 | |
339 | bytestream2_init(gb, buf, buf_size); |
340 | |
341 | c->pic->key_frame = 0; |
342 | c->pic->pict_type = AV_PICTURE_TYPE_P; |
343 | |
344 | // restore screen after cursor |
345 | if (c->screendta) { |
346 | int i; |
347 | w = c->cur_w; |
348 | if (c->width < c->cur_x + w) |
349 | w = c->width - c->cur_x; |
350 | h = c->cur_h; |
351 | if (c->height < c->cur_y + h) |
352 | h = c->height - c->cur_y; |
353 | dx = c->cur_x; |
354 | if (dx < 0) { |
355 | w += dx; |
356 | dx = 0; |
357 | } |
358 | dy = c->cur_y; |
359 | if (dy < 0) { |
360 | h += dy; |
361 | dy = 0; |
362 | } |
363 | if ((w > 0) && (h > 0)) { |
364 | outptr = c->pic->data[0] + dx * c->bpp2 + dy * c->pic->linesize[0]; |
365 | for (i = 0; i < h; i++) { |
366 | memcpy(outptr, c->screendta + i * c->cur_w * c->bpp2, |
367 | w * c->bpp2); |
368 | outptr += c->pic->linesize[0]; |
369 | } |
370 | } |
371 | } |
372 | bytestream2_skip(gb, 2); |
373 | chunks = bytestream2_get_be16(gb); |
374 | while (chunks--) { |
375 | if (bytestream2_get_bytes_left(gb) < 12) { |
376 | av_log(avctx, AV_LOG_ERROR, "Premature end of data!\n"); |
377 | return -1; |
378 | } |
379 | dx = bytestream2_get_be16(gb); |
380 | dy = bytestream2_get_be16(gb); |
381 | w = bytestream2_get_be16(gb); |
382 | h = bytestream2_get_be16(gb); |
383 | enc = bytestream2_get_be32(gb); |
384 | outptr = c->pic->data[0] + dx * c->bpp2 + dy * c->pic->linesize[0]; |
385 | size_left = bytestream2_get_bytes_left(gb); |
386 | switch (enc) { |
387 | case MAGIC_WMVd: // cursor |
388 | if (w*(int64_t)h*c->bpp2 > INT_MAX/2 - 2) { |
389 | av_log(avctx, AV_LOG_ERROR, "dimensions too large\n"); |
390 | return AVERROR_INVALIDDATA; |
391 | } |
392 | if (size_left < 2 + w * h * c->bpp2 * 2) { |
393 | av_log(avctx, AV_LOG_ERROR, |
394 | "Premature end of data! (need %i got %i)\n", |
395 | 2 + w * h * c->bpp2 * 2, size_left); |
396 | return AVERROR_INVALIDDATA; |
397 | } |
398 | bytestream2_skip(gb, 2); |
399 | c->cur_w = w; |
400 | c->cur_h = h; |
401 | c->cur_hx = dx; |
402 | c->cur_hy = dy; |
403 | if ((c->cur_hx > c->cur_w) || (c->cur_hy > c->cur_h)) { |
404 | av_log(avctx, AV_LOG_ERROR, |
405 | "Cursor hot spot is not in image: " |
406 | "%ix%i of %ix%i cursor size\n", |
407 | c->cur_hx, c->cur_hy, c->cur_w, c->cur_h); |
408 | c->cur_hx = c->cur_hy = 0; |
409 | } |
410 | if (c->cur_w * c->cur_h >= INT_MAX / c->bpp2) { |
411 | reset_buffers(c); |
412 | return AVERROR(EINVAL); |
413 | } else { |
414 | int screen_size = c->cur_w * c->cur_h * c->bpp2; |
415 | if ((ret = av_reallocp(&c->curbits, screen_size)) < 0 || |
416 | (ret = av_reallocp(&c->curmask, screen_size)) < 0 || |
417 | (ret = av_reallocp(&c->screendta, screen_size)) < 0) { |
418 | reset_buffers(c); |
419 | return ret; |
420 | } |
421 | } |
422 | load_cursor(c); |
423 | break; |
424 | case MAGIC_WMVe: // unknown |
425 | bytestream2_skip(gb, 2); |
426 | break; |
427 | case MAGIC_WMVf: // update cursor position |
428 | c->cur_x = dx - c->cur_hx; |
429 | c->cur_y = dy - c->cur_hy; |
430 | break; |
431 | case MAGIC_WMVg: // unknown |
432 | bytestream2_skip(gb, 10); |
433 | break; |
434 | case MAGIC_WMVh: // unknown |
435 | bytestream2_skip(gb, 4); |
436 | break; |
437 | case MAGIC_WMVi: // ServerInitialization struct |
438 | c->pic->key_frame = 1; |
439 | c->pic->pict_type = AV_PICTURE_TYPE_I; |
440 | depth = bytestream2_get_byte(gb); |
441 | if (depth != c->bpp) { |
442 | av_log(avctx, AV_LOG_INFO, |
443 | "Depth mismatch. Container %i bpp, " |
444 | "Frame data: %i bpp\n", |
445 | c->bpp, depth); |
446 | } |
447 | bytestream2_skip(gb, 1); |
448 | c->bigendian = bytestream2_get_byte(gb); |
449 | if (c->bigendian & (~1)) { |
450 | av_log(avctx, AV_LOG_INFO, |
451 | "Invalid header: bigendian flag = %i\n", c->bigendian); |
452 | return AVERROR_INVALIDDATA; |
453 | } |
454 | //skip the rest of pixel format data |
455 | bytestream2_skip(gb, 13); |
456 | break; |
457 | case MAGIC_WMVj: // unknown |
458 | bytestream2_skip(gb, 2); |
459 | break; |
460 | case 0x00000000: // raw rectangle data |
461 | if ((dx + w > c->width) || (dy + h > c->height)) { |
462 | av_log(avctx, AV_LOG_ERROR, |
463 | "Incorrect frame size: %ix%i+%ix%i of %ix%i\n", |
464 | w, h, dx, dy, c->width, c->height); |
465 | return AVERROR_INVALIDDATA; |
466 | } |
467 | if (size_left < w * h * c->bpp2) { |
468 | av_log(avctx, AV_LOG_ERROR, |
469 | "Premature end of data! (need %i got %i)\n", |
470 | w * h * c->bpp2, size_left); |
471 | return AVERROR_INVALIDDATA; |
472 | } |
473 | paint_raw(outptr, w, h, gb, c->bpp2, c->bigendian, |
474 | c->pic->linesize[0]); |
475 | break; |
476 | case 0x00000005: // HexTile encoded rectangle |
477 | if ((dx + w > c->width) || (dy + h > c->height)) { |
478 | av_log(avctx, AV_LOG_ERROR, |
479 | "Incorrect frame size: %ix%i+%ix%i of %ix%i\n", |
480 | w, h, dx, dy, c->width, c->height); |
481 | return AVERROR_INVALIDDATA; |
482 | } |
483 | res = decode_hextile(c, outptr, gb, w, h, c->pic->linesize[0]); |
484 | if (res < 0) |
485 | return res; |
486 | break; |
487 | default: |
488 | av_log(avctx, AV_LOG_ERROR, "Unsupported block type 0x%08X\n", enc); |
489 | chunks = 0; // leave chunks decoding loop |
490 | } |
491 | } |
492 | if (c->screendta) { |
493 | int i; |
494 | // save screen data before painting cursor |
495 | w = c->cur_w; |
496 | if (c->width < c->cur_x + w) |
497 | w = c->width - c->cur_x; |
498 | h = c->cur_h; |
499 | if (c->height < c->cur_y + h) |
500 | h = c->height - c->cur_y; |
501 | dx = c->cur_x; |
502 | if (dx < 0) { |
503 | w += dx; |
504 | dx = 0; |
505 | } |
506 | dy = c->cur_y; |
507 | if (dy < 0) { |
508 | h += dy; |
509 | dy = 0; |
510 | } |
511 | if ((w > 0) && (h > 0)) { |
512 | outptr = c->pic->data[0] + dx * c->bpp2 + dy * c->pic->linesize[0]; |
513 | for (i = 0; i < h; i++) { |
514 | memcpy(c->screendta + i * c->cur_w * c->bpp2, outptr, |
515 | w * c->bpp2); |
516 | outptr += c->pic->linesize[0]; |
517 | } |
518 | outptr = c->pic->data[0]; |
519 | put_cursor(outptr, c->pic->linesize[0], c, c->cur_x, c->cur_y); |
520 | } |
521 | } |
522 | *got_frame = 1; |
523 | if ((ret = av_frame_ref(data, c->pic)) < 0) |
524 | return ret; |
525 | |
526 | /* always report that the buffer was completely consumed */ |
527 | return buf_size; |
528 | } |
529 | |
530 | static av_cold int decode_init(AVCodecContext *avctx) |
531 | { |
532 | VmncContext * const c = avctx->priv_data; |
533 | |
534 | c->avctx = avctx; |
535 | c->width = avctx->width; |
536 | c->height = avctx->height; |
537 | c->bpp = avctx->bits_per_coded_sample; |
538 | |
539 | switch (c->bpp) { |
540 | case 8: |
541 | avctx->pix_fmt = AV_PIX_FMT_PAL8; |
542 | break; |
543 | case 16: |
544 | avctx->pix_fmt = AV_PIX_FMT_RGB555; |
545 | break; |
546 | case 24: |
547 | /* 24 bits is not technically supported, but some clients might |
548 | * mistakenly set it, so let's assume they actually meant 32 bits */ |
549 | c->bpp = 32; |
550 | case 32: |
551 | avctx->pix_fmt = AV_PIX_FMT_0RGB32; |
552 | break; |
553 | default: |
554 | av_log(avctx, AV_LOG_ERROR, "Unsupported bitdepth %i\n", c->bpp); |
555 | return AVERROR_INVALIDDATA; |
556 | } |
557 | c->bpp2 = c->bpp / 8; |
558 | |
559 | c->pic = av_frame_alloc(); |
560 | if (!c->pic) |
561 | return AVERROR(ENOMEM); |
562 | |
563 | return 0; |
564 | } |
565 | |
566 | static av_cold int decode_end(AVCodecContext *avctx) |
567 | { |
568 | VmncContext * const c = avctx->priv_data; |
569 | |
570 | av_frame_free(&c->pic); |
571 | |
572 | av_freep(&c->curbits); |
573 | av_freep(&c->curmask); |
574 | av_freep(&c->screendta); |
575 | return 0; |
576 | } |
577 | |
578 | AVCodec ff_vmnc_decoder = { |
579 | .name = "vmnc", |
580 | .long_name = NULL_IF_CONFIG_SMALL("VMware Screen Codec / VMware Video"), |
581 | .type = AVMEDIA_TYPE_VIDEO, |
582 | .id = AV_CODEC_ID_VMNC, |
583 | .priv_data_size = sizeof(VmncContext), |
584 | .init = decode_init, |
585 | .close = decode_end, |
586 | .decode = decode_frame, |
587 | .capabilities = AV_CODEC_CAP_DR1, |
588 | }; |
589 |