diff --git a/caca/attr.c b/caca/attr.c
index 4bd7fbb..be352ba 100644
--- a/caca/attr.c
+++ b/caca/attr.c
@@ -130,10 +130,13 @@ int caca_set_attr(caca_canvas_t *cv, uint32_t attr)
 int caca_put_attr(caca_canvas_t *cv, int x, int y, uint32_t attr)
 {
     uint32_t *curattr, *curchar;
+    int xmin, xmax;
 
     if(x < 0 || x >= (int)cv->width || y < 0 || y >= (int)cv->height)
         return 0;
 
+    xmin = xmax = x;
+
     curchar = cv->chars + x + y * cv->width;
     curattr = cv->attrs + x + y * cv->width;
 
@@ -143,9 +146,17 @@ int caca_put_attr(caca_canvas_t *cv, int x, int y, uint32_t attr)
         curattr[0] = attr;
 
     if(x && curchar[0] == CACA_MAGIC_FULLWIDTH)
+    {
         curattr[-1] = curattr[0];
+        xmin--;
+    }
     else if(x + 1 < (int)cv->width && curchar[1] == CACA_MAGIC_FULLWIDTH)
+    {
         curattr[1] = curattr[0];
+        xmax++;
+    }
+
+    caca_add_dirty_rectangle(cv, xmin, xmax, y, y);
 
     return 0;
 }
diff --git a/caca/caca_internals.h b/caca/caca_internals.h
index 75cab5a..ae3b27d 100644
--- a/caca/caca_internals.h
+++ b/caca/caca_internals.h
@@ -39,9 +39,6 @@ struct caca_frame
     int handlex, handley;
     uint32_t curattr;
 
-    /* Dirty rectangle */
-    int dirty_xmin, dirty_xmax, dirty_ymin, dirty_ymax;
-
     /* Frame name */
     char *name;
 };
@@ -61,6 +58,9 @@ struct caca_canvas
     int (*resize_callback)(void *);
     void *resize_data;
 
+    /* Dirty rectangle */
+    int dirty_xmin, dirty_xmax, dirty_ymin, dirty_ymax;
+
     /* Shortcut to the active frame information */
     int width, height;
     uint32_t *chars;
diff --git a/caca/canvas.c b/caca/canvas.c
index 7ddad66..a0cd8ae 100644
--- a/caca/canvas.c
+++ b/caca/canvas.c
@@ -89,14 +89,15 @@ caca_canvas_t * caca_create_canvas(int width, int height)
     cv->frames[0].handlex = cv->frames[0].handley = 0;
     cv->frames[0].curattr = 0;
     cv->frames[0].name = strdup("frame#00000000");
-    cv->frames[0].dirty_xmin = 0;
-    cv->frames[0].dirty_xmax = -1;
-    cv->frames[0].dirty_ymin = 0;
-    cv->frames[0].dirty_ymax = -1;
 
     _caca_load_frame_info(cv);
     caca_set_color_ansi(cv, CACA_DEFAULT, CACA_TRANSPARENT);
 
+    cv->dirty_xmin = 0;
+    cv->dirty_xmax = -1;
+    cv->dirty_ymin = 0;
+    cv->dirty_ymax = -1;
+
     cv->ff = NULL;
 
     if(caca_resize(cv, width, height) < 0)
@@ -331,10 +332,10 @@ uint8_t const * caca_get_canvas_attrs(caca_canvas_t const *cv)
 int caca_get_dirty_rectangle(caca_canvas_t *cv, int *xmin, int *xmax,
                              int *ymin, int *ymax)
 {
-    *xmin = cv->frames[cv->frame].dirty_xmin;
-    *xmax = cv->frames[cv->frame].dirty_xmax;
-    *ymin = cv->frames[cv->frame].dirty_ymin;
-    *ymax = cv->frames[cv->frame].dirty_ymax;
+    *xmin = cv->dirty_xmin;
+    *xmax = cv->dirty_xmax;
+    *ymin = cv->dirty_ymin;
+    *ymax = cv->dirty_ymax;
 
     return 0;
 }
@@ -371,17 +372,17 @@ int caca_add_dirty_rectangle(caca_canvas_t *cv, int xmin, int xmax,
     if(xmax < 0 || xmin >= cv->width || ymax < 0 || ymin >= cv->height)
         return 0;
 
-    if(xmin < cv->frames[cv->frame].dirty_xmin)
-        cv->frames[cv->frame].dirty_xmin = xmin;
+    if(xmin < cv->dirty_xmin)
+        cv->dirty_xmin = xmin;
 
-    if(xmax > cv->frames[cv->frame].dirty_xmax)
-        cv->frames[cv->frame].dirty_xmax = xmax;
+    if(xmax > cv->dirty_xmax)
+        cv->dirty_xmax = xmax;
 
-    if(ymin < cv->frames[cv->frame].dirty_ymin)
-        cv->frames[cv->frame].dirty_ymin = ymin;
+    if(ymin < cv->dirty_ymin)
+        cv->dirty_ymin = ymin;
 
-    if(ymax > cv->frames[cv->frame].dirty_ymax)
-        cv->frames[cv->frame].dirty_ymax = ymax;
+    if(ymax > cv->dirty_ymax)
+        cv->dirty_ymax = ymax;
 
     return 0;
 }
@@ -416,10 +417,10 @@ int caca_set_dirty_rectangle(caca_canvas_t *cv, int xmin, int xmax,
         ymax = -1;
     }
 
-    cv->frames[cv->frame].dirty_xmin = xmin;
-    cv->frames[cv->frame].dirty_xmax = xmax;
-    cv->frames[cv->frame].dirty_ymin = ymin;
-    cv->frames[cv->frame].dirty_ymax = ymax;
+    cv->dirty_xmin = xmin;
+    cv->dirty_xmax = xmax;
+    cv->dirty_ymin = ymin;
+    cv->dirty_ymax = ymax;
 
     return 0;
 }
@@ -555,6 +556,8 @@ int caca_resize(caca_canvas_t *cv, int width, int height)
                 }
             }
         }
+
+        caca_add_dirty_rectangle(cv, old_width, 0, width - 1, old_height - 1);
     }
     else
     {
@@ -594,8 +597,17 @@ int caca_resize(caca_canvas_t *cv, int width, int height)
                 attrs[old_height * width + x] = attr;
             }
         }
+
+        caca_add_dirty_rectangle(cv, 0, old_height, old_width - 1, height - 1);
     }
 
+    /* XXX: technically we should not worry about the dirty rectangle in
+     * the bottom-right corner, because we only handle one dirty rectangle,
+     * but in case the API changes later, we make sure this is handled. */
+    if(width > old_width && height > old_height)
+        caca_add_dirty_rectangle(cv, old_width, old_height,
+                                 width - 1, height - 1);
+
     /* Step 4: if new area is smaller, resize memory area now. */
     if(new_size < old_size)
     {
diff --git a/caca/frame.c b/caca/frame.c
index 1cffad9..022e840 100644
--- a/caca/frame.c
+++ b/caca/frame.c
@@ -64,10 +64,16 @@ int caca_set_frame(caca_canvas_t *cv, int id)
         return -1;
     }
 
+    /* Bail out if no operation is required */
+    if(id == cv->frame)
+        return 0;
+
     _caca_save_frame_info(cv);
     cv->frame = id;
     _caca_load_frame_info(cv);
 
+    caca_set_dirty_rectangle(cv, 0, 0, cv->width - 1, cv->height - 1);
+
     return 0;
 }
 
@@ -230,6 +236,7 @@ int caca_free_frame(caca_canvas_t *cv, int id)
     {
         cv->frame = 0;
         _caca_load_frame_info(cv);
+        caca_set_dirty_rectangle(cv, 0, 0, cv->width - 1, cv->height - 1);
     }
 
     return 0;
diff --git a/caca/graphics.c b/caca/graphics.c
index 27d35b3..6f13d5b 100644
--- a/caca/graphics.c
+++ b/caca/graphics.c
@@ -154,6 +154,9 @@ int caca_refresh_display(caca_display_t *dp)
 
     dp->drv.display(dp);
 
+    /* Invalidate the dirty rectangle */
+    caca_set_dirty_rectangle(dp->cv, -1, -1, -1, -1);
+
     /* Once the display is finished, we can ack resizes */
     if(dp->resize.resized)
     {
diff --git a/caca/string.c b/caca/string.c
index 627e764..4e70581 100644
--- a/caca/string.c
+++ b/caca/string.c
@@ -112,7 +112,7 @@ int caca_get_cursor_y(caca_canvas_t const *cv)
 int caca_put_char(caca_canvas_t *cv, int x, int y, uint32_t ch)
 {
     uint32_t *curchar, *curattr, attr;
-    int fullwidth;
+    int fullwidth, xmin, xmax;
 
     if(x >= (int)cv->width || y < 0 || y >= (int)cv->height)
         return 0;
@@ -135,10 +135,15 @@ int caca_put_char(caca_canvas_t *cv, int x, int y, uint32_t ch)
     curattr = cv->attrs + x + y * cv->width;
     attr = cv->curattr;
 
+    xmin = xmax = x;
+
     /* When overwriting the right part of a fullwidth character,
      * replace its left part with a space. */
     if(x && curchar[0] == CACA_MAGIC_FULLWIDTH)
+    {
         curchar[-1] = ' ';
+        xmin--;
+    }
 
     if(fullwidth)
     {
@@ -146,10 +151,15 @@ int caca_put_char(caca_canvas_t *cv, int x, int y, uint32_t ch)
             ch = ' ';
         else
         {
+            xmax++;
+
             /* When overwriting the left part of a fullwidth character,
              * replace its right part with a space. */
             if(x + 2 < (int)cv->width && curchar[2] == CACA_MAGIC_FULLWIDTH)
+            {
                 curchar[2] = ' ';
+                xmax++;
+            }
 
             curchar[1] = CACA_MAGIC_FULLWIDTH;
             curattr[1] = attr;
@@ -163,6 +173,8 @@ int caca_put_char(caca_canvas_t *cv, int x, int y, uint32_t ch)
             curchar[1] = ' ';
     }
 
+    caca_add_dirty_rectangle(cv, xmin, xmax, y, y);
+
     curchar[0] = ch;
     curattr[0] = attr;
 
@@ -307,6 +319,8 @@ int caca_clear_canvas(caca_canvas_t *cv)
         cv->attrs[n] = attr;
     }
 
+    caca_set_dirty_rectangle(cv, 0, 0, cv->width - 1, cv->height - 1);
+
     return 0;
 }
 
@@ -375,9 +389,9 @@ int caca_get_canvas_handle_y(caca_canvas_t const *cv)
  *  \return 0 in case of success, -1 if an error occurred.
  */
 int caca_blit(caca_canvas_t *dst, int x, int y,
-               caca_canvas_t const *src, caca_canvas_t const *mask)
+              caca_canvas_t const *src, caca_canvas_t const *mask)
 {
-    int i, j, starti, startj, endi, endj;
+    int i, j, starti, startj, endi, endj, stride, bleed_left, bleed_right;
 
     if(mask && (src->width != mask->width || src->height != mask->height))
     {
@@ -392,24 +406,32 @@ int caca_blit(caca_canvas_t *dst, int x, int y,
     startj = y < 0 ? -y : 0;
     endi = (x + src->width >= dst->width) ? dst->width - x : src->width;
     endj = (y + src->height >= dst->height) ? dst->height - y : src->height;
+    stride = endi - starti;
 
     if(starti > src->width || startj > src->height
         || starti >= endi || startj >= endj)
         return 0;
 
+    bleed_left = bleed_right = 0;
+
     for(j = startj; j < endj; j++)
     {
         int dstix = (j + y) * dst->width + starti + x;
         int srcix = j * src->width + starti;
-        int stride = endi - starti;
 
         /* FIXME: we are ignoring the mask here */
         if((starti + x) && dst->chars[dstix] == CACA_MAGIC_FULLWIDTH)
+        {
             dst->chars[dstix - 1] = ' ';
+            bleed_left = 1;
+        }
 
         if(endi + x < dst->width
                 && dst->chars[dstix + stride] == CACA_MAGIC_FULLWIDTH)
+        {
             dst->chars[dstix + stride] = ' ';
+            bleed_right = 1;
+        }
 
         if(mask)
         {
@@ -436,6 +458,9 @@ int caca_blit(caca_canvas_t *dst, int x, int y,
             dst->chars[dstix + stride - 1] = ' ';
     }
 
+    caca_add_dirty_rectangle(dst, starti + x - bleed_left, startj + y,
+                             endi + x - 1 + bleed_right, endj + y - 1);
+
     return 0;
 }
 
@@ -499,6 +524,9 @@ int caca_set_canvas_boundaries(caca_canvas_t *cv, int x, int y, int w, int h)
     caca_set_frame(cv, saved_f);
     _caca_load_frame_info(cv);
 
+    /* FIXME: this may be optimised somewhat */
+    caca_set_dirty_rectangle(cv, 0, 0, cv->width - 1, cv->height - 1);
+
     return 0;
 }
 
diff --git a/caca/transform.c b/caca/transform.c
index e19612b..1c8c359 100644
--- a/caca/transform.c
+++ b/caca/transform.c
@@ -54,6 +54,8 @@ int caca_invert(caca_canvas_t *cv)
         attrs++;
     }
 
+    caca_set_dirty_rectangle(cv, 0, 0, cv->width - 1, cv->height - 1);
+
     return 0;
 }
 
@@ -113,6 +115,8 @@ int caca_flip(caca_canvas_t *cv)
         }
     }
 
+    caca_set_dirty_rectangle(cv, 0, 0, cv->width - 1, cv->height - 1);
+
     return 0;
 }
 
@@ -158,6 +162,8 @@ int caca_flop(caca_canvas_t *cv)
             *ctop = flopchar(*ctop);
     }
 
+    caca_set_dirty_rectangle(cv, 0, 0, cv->width - 1, cv->height - 1);
+
     return 0;
 }
 
@@ -215,6 +221,8 @@ int caca_rotate_180(caca_canvas_t *cv)
         }
     }
 
+    caca_set_dirty_rectangle(cv, 0, 0, cv->width - 1, cv->height - 1);
+
     return 0;
 }
 
@@ -332,6 +340,8 @@ int caca_rotate_left(caca_canvas_t *cv)
     /* Reset the current frame shortcuts */
     _caca_load_frame_info(cv);
 
+    caca_set_dirty_rectangle(cv, 0, 0, cv->width - 1, cv->height - 1);
+
     return 0;
 }
 
@@ -449,6 +459,8 @@ int caca_rotate_right(caca_canvas_t *cv)
     /* Reset the current frame shortcuts */
     _caca_load_frame_info(cv);
 
+    caca_set_dirty_rectangle(cv, 0, 0, cv->width - 1, cv->height - 1);
+
     return 0;
 }
 
@@ -540,6 +552,8 @@ int caca_stretch_left(caca_canvas_t *cv)
     /* Reset the current frame shortcuts */
     _caca_load_frame_info(cv);
 
+    caca_set_dirty_rectangle(cv, 0, 0, cv->width - 1, cv->height - 1);
+
     return 0;
 }
 
@@ -631,6 +645,8 @@ int caca_stretch_right(caca_canvas_t *cv)
     /* Reset the current frame shortcuts */
     _caca_load_frame_info(cv);
 
+    caca_set_dirty_rectangle(cv, 0, 0, cv->width - 1, cv->height - 1);
+
     return 0;
 }