From 8b72afb6e22a169be74e1e71331b1b661a72ea21 Mon Sep 17 00:00:00 2001
From: Ken Sharp <ken.sharp@artifex.com>
Date: Thu, 13 Oct 2022 14:54:23 +0100
Subject: [PATCH 148/155] GhostPDL + GS - do not make PS colour spaces into PDF
 ones

This was reported (out of Bugzilla) by Robin Watts, noticed in passing
while debugging a different problem. Using the bug file (bears.pdf)
from Bug #705975 and the following command line:

-r1500 -sDEVICE=tiffsep -o /temp/out.tif /temp/bears.pdf

if the output file out.tif cannot be written (read-only or held open by
another application) then this caused an error. On a Memento build on
Windows it causes a corrupted block on exit, on Linux it was giving a
warning about attempting to free an unknown PDF object type.

In all cases the error occurred during the PostScript end of job restore
when there should be no PDF objects and the PDF interpreter context had
been released.

The problem was due to colour spaces, when we create a colour space in
PDF we set a callback, called when a colour space is freed, which does
extra work specific to the PDF interpreter. However we were ending up
adding these callbacks to colour spaces allocated by the PostScript
interpreter.

This comes about because gs_setcolorspace() does not alter the current
colour space if we attempt to switch to the same space. So in this case
the current colour space in the PostScript interpreter was DeviceGray
and the PDF interpreter then set DeviceGray, which did not result in a
new colour space in the graphics state. But we went ahead and assigned
the colour callbacks assuming that it had. When we finally freed that
space we would then attempt to handle it as a PDF space, but the PDF
context had long gone, resulting in an error.

To fix this we now check the colour space after calling gs_setcolorspace
to see if it has changed. If it has not we do not assign the PDF colour
callbacks; either we have previously done so (if its a PDF space) or it
is a PostScript space and we absolutely do not want to set the callbacks.

Why does this not show up normally ? Because we do a showpage after
every page, which does an initgraphics, which resets the PostScript
colour space, and at this point the PDF context is still valid. In order
to get this problem we currently need 'showpage' to return an error.

But it is still wrong, much better to avoid the situation.
---
 pdf/pdf_colour.c | 49 ++++++++++++++++++++++++++++++++++++++++++------
 1 file changed, 43 insertions(+), 6 deletions(-)

diff --git a/pdf/pdf_colour.c b/pdf/pdf_colour.c
index 9c7e74910..a403d9bec 100644
--- a/pdf/pdf_colour.c
+++ b/pdf/pdf_colour.c
@@ -342,6 +342,7 @@ static void pdfi_cspace_free_callback(gs_memory_t * mem, void *cs)
 int pdfi_gs_setgray(pdf_context *ctx, double d)
 {
     int code = 0;
+    gs_color_space *pcs = ctx->pgs->color[0].color_space;
 
     /* PDF Reference 1.7 p423, any colour operators in a CharProc, following a d1, should be ignored */
     if (ctx->text.inside_CharProc && ctx->text.CharProc_d_type != pdf_type3_d0)
@@ -353,6 +354,11 @@ int pdfi_gs_setgray(pdf_context *ctx, double d)
         code = gs_setcolorspace(ctx->pgs, ctx->page.DefaultGray_cs);
         if (code < 0)
             return code;
+        /* If we didn't change the colour space in the graphics state, do not attempt to
+         * set the callbacks, the current space might be inherited from PostScript.
+         */
+        if (pcs != ctx->pgs->color[0].color_space)
+            pdfi_set_colour_callback(ctx->pgs->color[0].color_space, ctx, pdfi_cspace_free_callback);
         cc.paint.values[0] = d;
         cc.pattern = 0;
         return gs_setcolor(ctx->pgs, &cc);
@@ -361,13 +367,18 @@ int pdfi_gs_setgray(pdf_context *ctx, double d)
         if (code < 0)
             return code;
     }
-    pdfi_set_colour_callback(ctx->pgs->color[0].color_space, ctx, pdfi_cspace_free_callback);
+    /* If we didn't change the colour space in the graphics state, do not attempt to
+     * set the callbacks, the current space might be inherited from PostScript.
+     */
+    if (pcs != ctx->pgs->color[0].color_space)
+        pdfi_set_colour_callback(ctx->pgs->color[0].color_space, ctx, pdfi_cspace_free_callback);
     return 0;
 }
 
 int pdfi_gs_setrgbcolor(pdf_context *ctx, double r, double g, double b)
 {
     int code = 0;
+    gs_color_space *pcs = ctx->pgs->color[0].color_space;
 
     /* PDF Reference 1.7 p423, any colour operators in a CharProc, following a d1, should be ignored */
     if (ctx->text.inside_CharProc && ctx->text.CharProc_d_type != pdf_type3_d0)
@@ -379,7 +390,11 @@ int pdfi_gs_setrgbcolor(pdf_context *ctx, double r, double g, double b)
         code = gs_setcolorspace(ctx->pgs, ctx->page.DefaultRGB_cs);
         if (code < 0)
             return code;
-        pdfi_set_colour_callback(ctx->pgs->color[0].color_space, ctx, NULL);
+        /* If we didn't change the colour space in the graphics state, do not attempt to
+         * set the callbacks, the current space might be inherited from PostScript.
+         */
+        if (pcs != ctx->pgs->color[0].color_space)
+            pdfi_set_colour_callback(ctx->pgs->color[0].color_space, ctx, pdfi_cspace_free_callback);
         cc.paint.values[0] = r;
         cc.paint.values[1] = g;
         cc.paint.values[2] = b;
@@ -389,7 +404,11 @@ int pdfi_gs_setrgbcolor(pdf_context *ctx, double r, double g, double b)
         code = gs_setrgbcolor(ctx->pgs, r, g, b);
         if (code < 0)
             return code;
-        pdfi_set_colour_callback(ctx->pgs->color[0].color_space, ctx, pdfi_cspace_free_callback);
+        /* If we didn't change the colour space in the graphics state, do not attempt to
+         * set the callbacks, the current space might be inherited from PostScript.
+         */
+        if (pcs != ctx->pgs->color[0].color_space)
+            pdfi_set_colour_callback(ctx->pgs->color[0].color_space, ctx, pdfi_cspace_free_callback);
     }
     return 0;
 }
@@ -397,6 +416,7 @@ int pdfi_gs_setrgbcolor(pdf_context *ctx, double r, double g, double b)
 static int pdfi_gs_setcmykcolor(pdf_context *ctx, double c, double m, double y, double k)
 {
     int code = 0;
+    gs_color_space *pcs = ctx->pgs->color[0].color_space;
 
     /* PDF Reference 1.7 p423, any colour operators in a CharProc, following a d1, should be ignored */
     if (ctx->text.inside_CharProc && ctx->text.CharProc_d_type != pdf_type3_d0)
@@ -408,6 +428,11 @@ static int pdfi_gs_setcmykcolor(pdf_context *ctx, double c, double m, double y,
         code = gs_setcolorspace(ctx->pgs, ctx->page.DefaultCMYK_cs);
         if (code < 0)
             return code;
+        /* If we didn't change the colour space in the graphics state, do not attempt to
+         * set the callbacks, the current space might be inherited from PostScript.
+         */
+        if (pcs != ctx->pgs->color[0].color_space)
+            pdfi_set_colour_callback(ctx->pgs->color[0].color_space, ctx, pdfi_cspace_free_callback);
         cc.paint.values[0] = c;
         cc.paint.values[1] = m;
         cc.paint.values[2] = y;
@@ -418,13 +443,19 @@ static int pdfi_gs_setcmykcolor(pdf_context *ctx, double c, double m, double y,
         code = gs_setcmykcolor(ctx->pgs, c, m, y, k);
         if (code < 0)
             return code;
+        /* If we didn't change the colour space in the graphics state, do not attempt to
+         * set the callbacks, the current space might be inherited from PostScript.
+         */
+        if (pcs != ctx->pgs->color[0].color_space)
+            pdfi_set_colour_callback(ctx->pgs->color[0].color_space, ctx, pdfi_cspace_free_callback);
     }
-    pdfi_set_colour_callback(ctx->pgs->color[0].color_space, ctx, pdfi_cspace_free_callback);
     return 0;
 }
 
 int pdfi_gs_setcolorspace(pdf_context *ctx, gs_color_space *pcs)
 {
+    gs_color_space *old_pcs = ctx->pgs->color[0].color_space;
+    int code = 0;
     /* If the target colour space is already the current colour space, don't
      * bother to do anything.
      */
@@ -433,8 +464,14 @@ int pdfi_gs_setcolorspace(pdf_context *ctx, gs_color_space *pcs)
         if (ctx->text.inside_CharProc && ctx->text.CharProc_d_type != pdf_type3_d0)
             return 0;
 
-        pdfi_set_colour_callback(pcs, ctx, pdfi_cspace_free_callback);
-        return gs_setcolorspace(ctx->pgs, pcs);
+        code = gs_setcolorspace(ctx->pgs, pcs);
+        if (code < 0)
+            return code;
+        /* If we didn't change the colour space in the graphics state, do not attempt to
+         * set the callbacks, the current space might be inherited from PostScript.
+         */
+        if (old_pcs != ctx->pgs->color[0].color_space)
+            pdfi_set_colour_callback(ctx->pgs->color[0].color_space, ctx, pdfi_cspace_free_callback);
     }
     return 0;
 }
-- 
2.30.4