Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 4 additions & 3 deletions Dockerfile.tmpl
Original file line number Diff line number Diff line change
Expand Up @@ -103,8 +103,9 @@ RUN mkdir -p /root/.EasyOCR/model && \
rm /root/.EasyOCR/model/craft_mlt_25k.zip && \
/tmp/clean-layer.sh

# Tesseract and some associated utility packages
RUN apt-get install tesseract-ocr -y && \
# Tesseract and some associated utility packages.
# poppler-utils provides pdftoppm, required by pdf2image.
RUN apt-get install tesseract-ocr poppler-utils -y && \
/tmp/clean-layer.sh

ENV TESSERACT_PATH=/usr/bin/tesseract \
Expand All @@ -126,7 +127,7 @@ RUN mkdir -p /root/.jupyter && touch /root/.jupyter/jupyter_nbconvert_config.py
RUN apt-get install libmagickwand-dev && \
/tmp/clean-layer.sh

# Override default imagemagick policies
# Restrictive imagemagick policy: blocks ImageTragick-class coders (URL, MVG, MSL, PS, PDF, ...)
ADD patches/imagemagick-policy.xml /etc/ImageMagick-6/policy.xml

# Add Kaggle module resolver
Expand Down
63 changes: 62 additions & 1 deletion patches/imagemagick-policy.xml
Original file line number Diff line number Diff line change
Expand Up @@ -7,4 +7,65 @@
name NMTOKEN #IMPLIED pattern CDATA #IMPLIED rights NMTOKEN #IMPLIED
stealth NMTOKEN #IMPLIED value CDATA #IMPLIED>
]>
<policymap></policymap>
<!--
Kaggle ImageMagick security policy.

Notebooks routinely open files from untrusted, user-contributed datasets, so
coders that can execute commands, fetch URLs, read arbitrary local files or
hand input to Ghostscript are disabled (ImageTragick, CVE-2016-3714 and
successors). Common raster formats (PNG, JPEG, GIF, TIFF, WebP, BMP, ...)
remain enabled.

To rasterize PDFs, use pdf2image (poppler) instead of Wand/ImageMagick.

Resource limits are intentionally generous compared to the distro defaults
so that large images (e.g. satellite or medical imagery) keep working, while
still bounding decompression bombs.
-->
<policymap>
<!-- Resource limits -->
<policy domain="resource" name="memory" value="8GiB"/>
<policy domain="resource" name="map" value="16GiB"/>
<policy domain="resource" name="width" value="128KP"/>
<policy domain="resource" name="height" value="128KP"/>
<policy domain="resource" name="area" value="4GP"/>
<policy domain="resource" name="disk" value="32GiB"/>

<!-- Network access (SSRF) -->
<policy domain="delegate" rights="none" pattern="URL" />
<policy domain="delegate" rights="none" pattern="HTTPS" />
<policy domain="delegate" rights="none" pattern="HTTP" />
<policy domain="coder" rights="none" pattern="URL" />
<policy domain="coder" rights="none" pattern="HTTPS" />
<policy domain="coder" rights="none" pattern="HTTP" />
<policy domain="coder" rights="none" pattern="FTP" />

<!-- Indirect reads of arbitrary files, e.g. label:@/etc/passwd -->
<policy domain="path" rights="none" pattern="@*" />

<!-- Scripting / file manipulation coders -->
<policy domain="coder" rights="none" pattern="EPHEMERAL" />
<policy domain="coder" rights="none" pattern="MVG" />
<policy domain="coder" rights="none" pattern="MSL" />
<policy domain="coder" rights="none" pattern="TEXT" />
<policy domain="coder" rights="none" pattern="SHOW" />
<policy domain="coder" rights="none" pattern="WIN" />
<policy domain="coder" rights="none" pattern="PLT" />

<!-- Ghostscript-backed coders -->
<policy domain="coder" rights="none" pattern="PS" />
<policy domain="coder" rights="none" pattern="PS2" />
<policy domain="coder" rights="none" pattern="PS3" />
<policy domain="coder" rights="none" pattern="EPS" />
<policy domain="coder" rights="none" pattern="EPSF" />
<policy domain="coder" rights="none" pattern="EPSI" />
<policy domain="coder" rights="none" pattern="EPI" />
<policy domain="coder" rights="none" pattern="EPT" />
<policy domain="coder" rights="none" pattern="PDF" />
<policy domain="coder" rights="none" pattern="PDFA" />
<policy domain="coder" rights="none" pattern="EPDF" />
<policy domain="coder" rights="none" pattern="AI" />
<policy domain="coder" rights="none" pattern="XPS" />
<policy domain="coder" rights="none" pattern="PCL" />
<policy domain="module" rights="none" pattern="{PS,PDF,XPS,PCL}" />
</policymap>
15 changes: 15 additions & 0 deletions tests/test_pdf2image.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
import unittest

from PIL import Image
from pdf2image import convert_from_path, pdfinfo_from_path


class TestPdf2Image(unittest.TestCase):
def test_convert_from_path(self):
info = pdfinfo_from_path('/input/tests/data/test.pdf')
pages = convert_from_path('/input/tests/data/test.pdf', dpi=72)

self.assertEqual(info['Pages'], len(pages))
self.assertIsInstance(pages[0], Image.Image)
self.assertGreater(pages[0].width, 0)
self.assertGreater(pages[0].height, 0)
18 changes: 8 additions & 10 deletions tests/test_pytesseract.py
Original file line number Diff line number Diff line change
@@ -1,15 +1,13 @@
import unittest
import io

import pytesseract
import numpy as np
from PIL import Image
from wand.image import Image as wandimage
from pdf2image import convert_from_path

class TestPytesseract(unittest.TestCase):
def test_tesseract(self):
# Open pdf with Wand
with wandimage(filename='/input/tests/data/test.pdf') as wand_image:
img_buffer = np.asarray(bytearray(wand_image.make_blob(format='png')), dtype='uint8')
bytesio = io.BytesIO(img_buffer)
test_string = pytesseract.image_to_string(Image.open(bytesio))
self.assertTrue(type(test_string) == str)
# Rasterize the pdf with pdf2image (poppler). ImageMagick's PDF coder is
# disabled by patches/imagemagick-policy.xml.
pages = convert_from_path('/input/tests/data/test.pdf')
self.assertGreater(len(pages), 0)
test_string = pytesseract.image_to_string(pages[0])
self.assertTrue(type(test_string) == str)
58 changes: 58 additions & 0 deletions tests/test_wand.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
import tempfile
import unittest

from wand.exceptions import PolicyError
from wand.image import Image


class TestWand(unittest.TestCase):
def test_read_png(self):
with Image(filename='/input/tests/data/dot.png') as img:
self.assertGreater(img.width, 0)
self.assertGreater(len(img.make_blob(format='jpeg')), 0)

def test_read_jpeg(self):
with Image(filename='/input/tests/data/face.jpg') as img:
self.assertGreater(img.width, 0)

def test_read_tiff(self):
with Image(filename='/input/tests/data/test.tif') as img:
self.assertGreater(img.width, 0)


class TestImageMagickPolicy(unittest.TestCase):
"""Checks that patches/imagemagick-policy.xml blocks ImageTragick-class coders."""

def assertBlocked(self, **kwargs):
with self.assertRaises(PolicyError):
with Image(**kwargs):
pass

def test_pdf_blocked(self):
self.assertBlocked(filename='/input/tests/data/test.pdf')

def test_pdf_blob_blocked(self):
# Content sniffing must not bypass the policy.
with open('/input/tests/data/test.pdf', 'rb') as f:
self.assertBlocked(blob=f.read())

def test_postscript_blocked(self):
self.assertBlocked(blob=b'%!PS-Adobe-3.0\nshowpage\n')

def test_indirect_file_read_blocked(self):
with tempfile.NamedTemporaryFile('w', suffix='.txt') as f:
f.write('secret')
f.flush()
self.assertBlocked(filename='label:@' + f.name)

def test_text_blocked(self):
self.assertBlocked(filename='text:/input/tests/data/text.txt')

def test_mvg_blocked(self):
self.assertBlocked(blob=b'push graphic-context\nviewbox 0 0 1 1\npop graphic-context\n', format='mvg')

def test_msl_blocked(self):
self.assertBlocked(blob=b'<?xml version="1.0"?><image></image>', format='msl')

def test_url_blocked(self):
self.assertBlocked(filename='https://www.kaggle.com/static/images/logos/kaggle-logo-transparent-300.png')
Loading