Fix CVE-2017-17858:

https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2017-17858
https://bugs.ghostscript.com/show_bug.cgi?id=698819
https://github.com/mzet-/Security-Advisories/blob/master/mzet-adv-2017-01.md

Patch copied from upstream source repository:

https://git.ghostscript.com/?p=mupdf.git;a=commit;h=55c3f68d638ac1263a386e0aaa004bb6e8bde731

From 55c3f68d638ac1263a386e0aaa004bb6e8bde731 Mon Sep 17 00:00:00 2001
From: Sebastian Rasmussen <sebras@gmail.com>
Date: Mon, 11 Dec 2017 14:09:15 +0100
Subject: [PATCH] Bugs 698804/698810/698811: Keep PDF object numbers below
 limit.

This ensures that:
 * xref tables with objects pointers do not grow out of bounds.
 * other readers, e.g. Adobe Acrobat can parse PDFs written by mupdf.
---
 include/mupdf/pdf/object.h |  3 +++
 source/pdf/pdf-repair.c    |  5 +----
 source/pdf/pdf-xref.c      | 21 ++++++++++++---------
 3 files changed, 16 insertions(+), 13 deletions(-)

diff --git a/include/mupdf/pdf/object.h b/include/mupdf/pdf/object.h
index 21ed8595..4177112b 100644
--- a/include/mupdf/pdf/object.h
+++ b/include/mupdf/pdf/object.h
@@ -3,6 +3,9 @@
 
 typedef struct pdf_document_s pdf_document;
 
+/* Defined in PDF 1.7 according to Acrobat limit. */
+#define PDF_MAX_OBJECT_NUMBER 8388607
+
 /*
  * Dynamic objects.
  * The same type of objects as found in PDF and PostScript.
diff --git a/source/pdf/pdf-repair.c b/source/pdf/pdf-repair.c
index ca149bd3..0c29758e 100644
--- a/source/pdf/pdf-repair.c
+++ b/source/pdf/pdf-repair.c
@@ -6,9 +6,6 @@
 
 /* Scan file for objects and reconstruct xref table */
 
-/* Define in PDF 1.7 to be 8388607, but mupdf is more lenient. */
-#define MAX_OBJECT_NUMBER (10 << 20)
-
 struct entry
 {
 	int num;
@@ -436,7 +433,7 @@ pdf_repair_xref(fz_context *ctx, pdf_document *doc)
 					break;
 				}
 
-				if (num <= 0 || num > MAX_OBJECT_NUMBER)
+				if (num <= 0 || num > PDF_MAX_OBJECT_NUMBER)
 				{
 					fz_warn(ctx, "ignoring object with invalid object number (%d %d R)", num, gen);
 					goto have_next_token;
diff --git a/source/pdf/pdf-xref.c b/source/pdf/pdf-xref.c
index 00586dbd..6284e70b 100644
--- a/source/pdf/pdf-xref.c
+++ b/source/pdf/pdf-xref.c
@@ -868,11 +868,12 @@ pdf_read_old_xref(fz_context *ctx, pdf_document *doc, pdf_lexbuf *buf)
 			fz_seek(ctx, file, -(2 + (int)strlen(s)), SEEK_CUR);
 		}
 
-		if (ofs < 0)
-			fz_throw(ctx, FZ_ERROR_GENERIC, "out of range object num in xref: %d", (int)ofs);
-		if (ofs > INT64_MAX - len)
-			fz_throw(ctx, FZ_ERROR_GENERIC, "xref section object numbers too big");
-
+		if (ofs < 0 || ofs > PDF_MAX_OBJECT_NUMBER
+				|| len < 0 || len > PDF_MAX_OBJECT_NUMBER
+				|| ofs + len - 1 > PDF_MAX_OBJECT_NUMBER)
+		{
+			fz_throw(ctx, FZ_ERROR_GENERIC, "xref subsection object numbers are out of range");
+		}
 		/* broken pdfs where size in trailer undershoots entries in xref sections */
 		if (ofs + len > xref_len)
 		{
@@ -933,10 +934,8 @@ pdf_read_new_xref_section(fz_context *ctx, pdf_document *doc, fz_stream *stm, in
 	pdf_xref_entry *table;
 	int i, n;
 
-	if (i0 < 0 || i1 < 0 || i0 > INT_MAX - i1)
-		fz_throw(ctx, FZ_ERROR_GENERIC, "negative xref stream entry index");
-	//if (i0 + i1 > pdf_xref_len(ctx, doc))
-	//	fz_throw(ctx, FZ_ERROR_GENERIC, "xref stream has too many entries");
+	if (i0 < 0 || i0 > PDF_MAX_OBJECT_NUMBER || i1 < 0 || i1 > PDF_MAX_OBJECT_NUMBER || i0 + i1 - 1 > PDF_MAX_OBJECT_NUMBER)
+		fz_throw(ctx, FZ_ERROR_GENERIC, "xref subsection object numbers are out of range");
 
 	table = pdf_xref_find_subsection(ctx, doc, i0, i1);
 	for (i = i0; i < i0 + i1; i++)
@@ -2086,6 +2085,10 @@ pdf_create_object(fz_context *ctx, pdf_document *doc)
 	/* TODO: reuse free object slots by properly linking free object chains in the ofs field */
 	pdf_xref_entry *entry;
 	int num = pdf_xref_len(ctx, doc);
+
+	if (num > PDF_MAX_OBJECT_NUMBER)
+		fz_throw(ctx, FZ_ERROR_GENERIC, "too many objects stored in pdf");
+
 	entry = pdf_get_incremental_xref_entry(ctx, doc, num);
 	entry->type = 'f';
 	entry->ofs = -1;
-- 
2.16.1