diff --git a/Dockerfile.tmpl b/Dockerfile.tmpl index 5de9b369..3904d099 100644 --- a/Dockerfile.tmpl +++ b/Dockerfile.tmpl @@ -103,8 +103,9 @@ RUN mkdir -p /root/.EasyOCR/model && \ rm /root/.EasyOCR/model/craft_mlt_25k.zip && \ /tmp/clean-layer.sh -# Tesseract and some associated utility packages -RUN apt-get install tesseract-ocr -y && \ +# Tesseract and some associated utility packages. +# poppler-utils provides pdftoppm, required by pdf2image. +RUN apt-get install tesseract-ocr poppler-utils -y && \ /tmp/clean-layer.sh ENV TESSERACT_PATH=/usr/bin/tesseract \ @@ -126,7 +127,7 @@ RUN mkdir -p /root/.jupyter && touch /root/.jupyter/jupyter_nbconvert_config.py RUN apt-get install libmagickwand-dev && \ /tmp/clean-layer.sh -# Override default imagemagick policies +# Restrictive imagemagick policy: blocks ImageTragick-class coders (URL, MVG, MSL, PS, PDF, ...) ADD patches/imagemagick-policy.xml /etc/ImageMagick-6/policy.xml # Add Kaggle module resolver diff --git a/patches/imagemagick-policy.xml b/patches/imagemagick-policy.xml index 981fb59f..4d85aff4 100644 --- a/patches/imagemagick-policy.xml +++ b/patches/imagemagick-policy.xml @@ -7,4 +7,65 @@ name NMTOKEN #IMPLIED pattern CDATA #IMPLIED rights NMTOKEN #IMPLIED stealth NMTOKEN #IMPLIED value CDATA #IMPLIED> ]> - \ No newline at end of file + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/test_pdf2image.py b/tests/test_pdf2image.py new file mode 100644 index 00000000..366f18dd --- /dev/null +++ b/tests/test_pdf2image.py @@ -0,0 +1,15 @@ +import unittest + +from PIL import Image +from pdf2image import convert_from_path, pdfinfo_from_path + + +class TestPdf2Image(unittest.TestCase): + def test_convert_from_path(self): + info = pdfinfo_from_path('/input/tests/data/test.pdf') + pages = convert_from_path('/input/tests/data/test.pdf', dpi=72) + + self.assertEqual(info['Pages'], len(pages)) + self.assertIsInstance(pages[0], Image.Image) + self.assertGreater(pages[0].width, 0) + self.assertGreater(pages[0].height, 0) diff --git a/tests/test_pytesseract.py b/tests/test_pytesseract.py index 353b8adb..95417877 100644 --- a/tests/test_pytesseract.py +++ b/tests/test_pytesseract.py @@ -1,15 +1,13 @@ import unittest -import io + import pytesseract -import numpy as np -from PIL import Image -from wand.image import Image as wandimage +from pdf2image import convert_from_path class TestPytesseract(unittest.TestCase): def test_tesseract(self): - # Open pdf with Wand - with wandimage(filename='/input/tests/data/test.pdf') as wand_image: - img_buffer = np.asarray(bytearray(wand_image.make_blob(format='png')), dtype='uint8') - bytesio = io.BytesIO(img_buffer) - test_string = pytesseract.image_to_string(Image.open(bytesio)) - self.assertTrue(type(test_string) == str) + # Rasterize the pdf with pdf2image (poppler). ImageMagick's PDF coder is + # disabled by patches/imagemagick-policy.xml. + pages = convert_from_path('/input/tests/data/test.pdf') + self.assertGreater(len(pages), 0) + test_string = pytesseract.image_to_string(pages[0]) + self.assertTrue(type(test_string) == str) diff --git a/tests/test_wand.py b/tests/test_wand.py new file mode 100644 index 00000000..113d31ce --- /dev/null +++ b/tests/test_wand.py @@ -0,0 +1,58 @@ +import tempfile +import unittest + +from wand.exceptions import PolicyError +from wand.image import Image + + +class TestWand(unittest.TestCase): + def test_read_png(self): + with Image(filename='/input/tests/data/dot.png') as img: + self.assertGreater(img.width, 0) + self.assertGreater(len(img.make_blob(format='jpeg')), 0) + + def test_read_jpeg(self): + with Image(filename='/input/tests/data/face.jpg') as img: + self.assertGreater(img.width, 0) + + def test_read_tiff(self): + with Image(filename='/input/tests/data/test.tif') as img: + self.assertGreater(img.width, 0) + + +class TestImageMagickPolicy(unittest.TestCase): + """Checks that patches/imagemagick-policy.xml blocks ImageTragick-class coders.""" + + def assertBlocked(self, **kwargs): + with self.assertRaises(PolicyError): + with Image(**kwargs): + pass + + def test_pdf_blocked(self): + self.assertBlocked(filename='/input/tests/data/test.pdf') + + def test_pdf_blob_blocked(self): + # Content sniffing must not bypass the policy. + with open('/input/tests/data/test.pdf', 'rb') as f: + self.assertBlocked(blob=f.read()) + + def test_postscript_blocked(self): + self.assertBlocked(blob=b'%!PS-Adobe-3.0\nshowpage\n') + + def test_indirect_file_read_blocked(self): + with tempfile.NamedTemporaryFile('w', suffix='.txt') as f: + f.write('secret') + f.flush() + self.assertBlocked(filename='label:@' + f.name) + + def test_text_blocked(self): + self.assertBlocked(filename='text:/input/tests/data/text.txt') + + def test_mvg_blocked(self): + self.assertBlocked(blob=b'push graphic-context\nviewbox 0 0 1 1\npop graphic-context\n', format='mvg') + + def test_msl_blocked(self): + self.assertBlocked(blob=b'', format='msl') + + def test_url_blocked(self): + self.assertBlocked(filename='https://www.kaggle.com/static/images/logos/kaggle-logo-transparent-300.png')