From c42ce95a597bcb03738182f77b27e55f365c6109 Mon Sep 17 00:00:00 2001 From: Andrew Volozhanin Date: Wed, 17 Jun 2015 20:13:24 +0500 Subject: [PATCH 1/5] Add layout option to keep layout during text extraction It passed -layout option to pdftotext. --- lib/docsplit/text_extractor.rb | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/lib/docsplit/text_extractor.rb b/lib/docsplit/text_extractor.rb index 985abdd..93973f6 100644 --- a/lib/docsplit/text_extractor.rb +++ b/lib/docsplit/text_extractor.rb @@ -102,17 +102,26 @@ def run(command) result end + # Run pdftotext command + def run_pdftotext(pdf, text_path, options=[]) + options << '-enc UTF-8' + options << '-layout' if @keep_layout + + run "pdftotext #{options.join(' ')} #{ESCAPE[pdf]} #{ESCAPE[text_path]} 2>&1" + end + # Extract the full contents of a pdf as a single file, directly. def extract_full(pdf) text_path = File.join(@output, "#{@pdf_name}.txt") - run "pdftotext -enc UTF-8 #{ESCAPE[pdf]} #{ESCAPE[text_path]} 2>&1" + run_pdftotext pdf, text_path end # Extract the contents of a single page of text, directly, adding it to # the `@pages_to_ocr` list if the text length is inadequate. def extract_page(pdf, page) text_path = File.join(@output, "#{@pdf_name}_#{page}.txt") - run "pdftotext -enc UTF-8 -f #{page} -l #{page} #{ESCAPE[pdf]} #{ESCAPE[text_path]} 2>&1" + run_pdftotext pdf, text_path, ["-f #{page}", "-l #{page}"] + unless @forbid_ocr @pages_to_ocr.push(page) if File.read(text_path).length < MIN_TEXT_PER_PAGE end @@ -126,6 +135,7 @@ def extract_options(options) @language = options[:language] || 'eng' @clean_ocr = (!(options[:clean] == false) and @language == 'eng') @detect_orientation = ((options[:detect_orientation] != false) and DEPENDENCIES[:osd]) + @keep_layout = options.fetch(:layout, false) end end From ee355d5fa4f70400396d52a3d048ef359267a1ec Mon Sep 17 00:00:00 2001 From: "T. Kim Nguyen" Date: Thu, 12 Apr 2018 11:33:49 -0500 Subject: [PATCH 2/5] update pdftk installer URL for Mac --- index.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/index.html b/index.html index a5c37e4..ccbcb95 100755 --- a/index.html +++ b/index.html @@ -149,7 +149,7 @@

Installation & Dependencies

(Optional) Install pdftk. On Linux, use aptitude, apt-get or yum:
aptitude install pdftk
- On the Mac, you can download a recent installer for the binary. + On the Mac, you can download a recent installer for the binary. Without pdftk installed, you can use Docsplit, but won't be able to split apart a multi-page PDF into single-page PDFs. From 6418e2531d87245a99d21a1ac942e46e53a41532 Mon Sep 17 00:00:00 2001 From: Anuja Ware-Mahajan Date: Mon, 8 Nov 2021 15:58:21 +0530 Subject: [PATCH 3/5] Documentaion link correction --- README | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README b/README index 34ce202..81374da 100755 --- a/README +++ b/README @@ -15,7 +15,7 @@ gem install docsplit For documentation, usage, and examples, see: - http://documentcloud.github.com/docsplit/ + https://documentcloud.github.io/docsplit/ To suggest a feature or report a bug: http://github.com/documentcloud/docsplit/issues/ From 990b0452b70b2d21e97157e60538e79c47723114 Mon Sep 17 00:00:00 2001 From: = <=> Date: Fri, 5 May 2023 13:23:34 -0700 Subject: [PATCH 4/5] Fix deprecated method File.exists? to File.exist? --- lib/docsplit/page_extractor.rb | 4 ++-- lib/docsplit/text_extractor.rb | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/docsplit/page_extractor.rb b/lib/docsplit/page_extractor.rb index 145c980..cb84e34 100644 --- a/lib/docsplit/page_extractor.rb +++ b/lib/docsplit/page_extractor.rb @@ -10,8 +10,8 @@ def extract(pdfs, opts) [pdfs].flatten.each do |pdf| pdf_name = File.basename(pdf, File.extname(pdf)) page_path = ESCAPE[File.join(@output, "#{pdf_name}")] + "_%d.pdf" - FileUtils.mkdir_p @output unless File.exists?(@output) - + FileUtils.mkdir_p @output unless File.exist?(@output) + cmd = if DEPENDENCIES[:pdftailor] # prefer pdftailor, but keep pdftk for backwards compatability "pdftailor unstitch --output #{page_path} #{ESCAPE[pdf]} 2>&1" else diff --git a/lib/docsplit/text_extractor.rb b/lib/docsplit/text_extractor.rb index 93973f6..27e4d29 100644 --- a/lib/docsplit/text_extractor.rb +++ b/lib/docsplit/text_extractor.rb @@ -28,7 +28,7 @@ def initialize # Extract text from a list of PDFs. def extract(pdfs, opts) extract_options opts - FileUtils.mkdir_p @output unless File.exists?(@output) + FileUtils.mkdir_p @output unless File.exist?(@output) [pdfs].flatten.each do |pdf| @pdf_name = File.basename(pdf, File.extname(pdf)) pages = (@pages == 'all') ? 1..Docsplit.extract_length(pdf) : @pages From 6127e3912b8db94ed84dca6be5622d3d5ec0d879 Mon Sep 17 00:00:00 2001 From: = <=> Date: Fri, 5 May 2023 13:36:07 -0700 Subject: [PATCH 5/5] FIx more instances of deprecated File.exists? --- lib/docsplit/image_extractor.rb | 4 ++-- lib/docsplit/page_extractor.rb | 2 +- lib/docsplit/pdf_extractor.rb | 30 +++++++++++++++--------------- lib/docsplit/text_extractor.rb | 2 +- test/test_helper.rb | 2 +- 5 files changed, 20 insertions(+), 20 deletions(-) diff --git a/lib/docsplit/image_extractor.rb b/lib/docsplit/image_extractor.rb index 8c29bbc..8bc4d1d 100755 --- a/lib/docsplit/image_extractor.rb +++ b/lib/docsplit/image_extractor.rb @@ -33,7 +33,7 @@ def convert(pdf, size, format, previous=nil) directory = directory_for(size) pages = @pages || '1-' + Docsplit.extract_length(pdf).to_s escaped_pdf = ESCAPE[pdf] - FileUtils.mkdir_p(directory) unless File.exists?(directory) + FileUtils.mkdir_p(directory) unless File.exist?(directory) common = "#{MEMORY_ARGS} -density #{@density} #{resize_arg(size)} #{quality_arg(format)}" if previous FileUtils.cp(Dir[directory_for(previous) + '/*'], directory) @@ -48,7 +48,7 @@ def convert(pdf, size, format, previous=nil) end end ensure - FileUtils.remove_entry_secure tempdir if File.exists?(tempdir) + FileUtils.remove_entry_secure tempdir if File.exist?(tempdir) end diff --git a/lib/docsplit/page_extractor.rb b/lib/docsplit/page_extractor.rb index cb84e34..0aef939 100644 --- a/lib/docsplit/page_extractor.rb +++ b/lib/docsplit/page_extractor.rb @@ -18,7 +18,7 @@ def extract(pdfs, opts) "pdftk #{ESCAPE[pdf]} burst output #{page_path} 2>&1" end result = `#{cmd}`.chomp - FileUtils.rm('doc_data.txt') if File.exists?('doc_data.txt') + FileUtils.rm('doc_data.txt') if File.exist?('doc_data.txt') raise ExtractionFailed, result if $? != 0 result end diff --git a/lib/docsplit/pdf_extractor.rb b/lib/docsplit/pdf_extractor.rb index 21861e2..a479265 100644 --- a/lib/docsplit/pdf_extractor.rb +++ b/lib/docsplit/pdf_extractor.rb @@ -16,7 +16,7 @@ def osx? def linux? !!HOST_OS.match(/linux/i) end - + # The first line of the help output holds the name and version number # of the office software to be used for extraction. def version_string @@ -35,10 +35,10 @@ def libre_office? def open_office? !!version_string.match(/^OpenOffice.org/) end - + # A set of default locations to search for office software # These have been extracted from JODConverter. Each listed - # path should contain a directory "program" which in turn + # path should contain a directory "program" which in turn # contains the "soffice" executable. # see: https://github.com/mirkonasato/jodconverter/blob/master/jodconverter-core/src/main/java/org/artofsolving/jodconverter/office/OfficeUtils.java#L63-L91 def office_search_paths @@ -69,7 +69,7 @@ def office_search_paths end search_paths end - + # Identify the path to a working office executable. def office_executable paths = office_search_paths @@ -78,10 +78,10 @@ def office_executable # raise an error if that path isn't valid, otherwise, add # it to the front of our search paths. if ENV['OFFICE_PATH'] - raise ArgumentError, "No such file or directory #{ENV['OFFICE_PATH']}" unless File.exists? ENV['OFFICE_PATH'] + raise ArgumentError, "No such file or directory #{ENV['OFFICE_PATH']}" unless File.exist? ENV['OFFICE_PATH'] paths.unshift(ENV['OFFICE_PATH']) end - + # The location of the office executable is OS dependent path_pieces = ["soffice"] if windows? @@ -91,15 +91,15 @@ def office_executable else path_pieces += [["program", "soffice"]] end - + # Search for the first suitable office executable # and short circuit an executable is found. paths.each do |path| - if File.exists? path + if File.exist? path @@executable ||= path unless File.directory? path path_pieces.each do |pieces| check_path = File.join(path, pieces) - @@executable ||= check_path if File.exists? check_path + @@executable ||= check_path if File.exist? check_path end end break if @@executable @@ -107,16 +107,16 @@ def office_executable raise OfficeNotFound, "No office software found" unless @@executable @@executable end - + # Used to specify the office location for JODConverter def office_path File.dirname(File.dirname(office_executable)) end - + # Convert documents to PDF. def extract(docs, opts) out = opts[:output] || '.' - FileUtils.mkdir_p out unless File.exists?(out) + FileUtils.mkdir_p out unless File.exist?(out) [docs].flatten.each do |doc| ext = File.extname(doc) basename = File.basename(doc, ext) @@ -128,7 +128,7 @@ def extract(docs, opts) if libre_office? # Set the LibreOffice user profile, so that parallel uses of cloudcrowd don't trip over each other. ENV['SYSUSERCONFIG']="file://#{File.expand_path(escaped_out)}" - + options = "--headless --invisible --norestore --nolockcheck --convert-to pdf --outdir #{escaped_out} #{escaped_doc}" cmd = "#{office_executable} #{options} 2>&1" result = `#{cmd}`.chomp @@ -147,9 +147,9 @@ def extract(docs, opts) LOGGING = "-Djava.util.logging.config.file=#{ESCAPED_ROOT}/vendor/logging.properties" HEADLESS = "-Djava.awt.headless=true" - + private - + # Runs a Java command, with quieted logging, and the classpath set properly. def run_jod(command, pdfs, opts, return_output=false) diff --git a/lib/docsplit/text_extractor.rb b/lib/docsplit/text_extractor.rb index 27e4d29..f3390e8 100644 --- a/lib/docsplit/text_extractor.rb +++ b/lib/docsplit/text_extractor.rb @@ -80,7 +80,7 @@ def extract_from_ocr(pdf, pages) clean_text(base_path + '.txt') if @clean_ocr end ensure - FileUtils.remove_entry_secure tempdir if File.exists?(tempdir) + FileUtils.remove_entry_secure tempdir if File.exist?(tempdir) end diff --git a/test/test_helper.rb b/test/test_helper.rb index 9c37b2b..2357c5a 100755 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -10,7 +10,7 @@ class Minitest::Test OUTPUT = 'test/output' def clear_output - FileUtils.rm_r(OUTPUT) if File.exists?(OUTPUT) + FileUtils.rm_r(OUTPUT) if File.exist?(OUTPUT) end def teardown