Detect text from picture

This commit is contained in:
Justin Mazzocchi 2021-03-27 00:31:13 -07:00
parent 5cc2a5a3ec
commit fd18fab063
No known key found for this signature in database
GPG key ID: E223E6937AAFB01C
2 changed files with 112 additions and 3 deletions

View file

@ -81,6 +81,7 @@
"attachment.edit.description" = "Describe for the visually impaired";
"attachment.edit.description.audio" = "Describe for people with hearing loss";
"attachment.edit.description.video" = "Describe for people with hearing loss or visual impairment";
"attachment.edit.detect-text-from-picture" = "Detect text from picture";
"attachment.edit.title" = "Edit media";
"attachment.edit.thumbnail.prompt" = "Drag the circle on the preview to choose the focal point which will always be in view on all thumbnails";
"attachment.sensitive-content" = "Sensitive content";

View file

@ -2,10 +2,15 @@
import AVKit
import Combine
import SDWebImage
import UIKit
import ViewModels
import Vision
final class EditAttachmentViewController: UIViewController {
private let textView = UITextView()
private let detectTextFromPictureButton = UIButton(type: .system)
private let detectTextFromPictureProgressView = UIProgressView()
private let viewModel: AttachmentViewModel
private let parentViewModel: CompositionViewModel
private var cancellables = Set<AnyCancellable>()
@ -88,8 +93,6 @@ final class EditAttachmentViewController: UIViewController {
describeLabel.text = NSLocalizedString("attachment.edit.description", comment: "")
}
let textView = UITextView()
stackView.addArrangedSubview(textView)
textView.adjustsFontForContentSizeCategory = true
textView.font = .preferredFont(forTextStyle: .body)
@ -100,12 +103,31 @@ final class EditAttachmentViewController: UIViewController {
textView.text = viewModel.editingDescription
textView.accessibilityLabel = describeLabel.text
let lowerStackView = UIStackView()
stackView.addArrangedSubview(lowerStackView)
lowerStackView.spacing = .defaultSpacing
let remainingCharactersLabel = UILabel()
stackView.addArrangedSubview(remainingCharactersLabel)
lowerStackView.addArrangedSubview(remainingCharactersLabel)
remainingCharactersLabel.adjustsFontForContentSizeCategory = true
remainingCharactersLabel.font = .preferredFont(forTextStyle: .subheadline)
lowerStackView.addArrangedSubview(detectTextFromPictureButton)
detectTextFromPictureButton.setTitle(
NSLocalizedString("attachment.edit.detect-text-from-picture", comment: ""),
for: .normal)
detectTextFromPictureButton.titleLabel?.adjustsFontSizeToFitWidth = true
detectTextFromPictureButton.titleLabel?.numberOfLines = 0
detectTextFromPictureButton.addAction(
UIAction { [weak self] _ in self?.detectTextFromPicture() },
for: .touchUpInside)
detectTextFromPictureButton.isHidden = viewModel.attachment.type != .image
stackView.addArrangedSubview(detectTextFromPictureProgressView)
detectTextFromPictureProgressView.isHidden = true
NSLayoutConstraint.activate([
stackView.leadingAnchor.constraint(equalTo: view.layoutMarginsGuide.leadingAnchor),
stackView.topAnchor.constraint(equalTo: view.layoutMarginsGuide.topAnchor, constant: .defaultSpacing),
@ -157,3 +179,89 @@ extension EditAttachmentViewController: UITextViewDelegate {
viewModel.editingDescription = textView.text
}
}
private extension EditAttachmentViewController {
enum TextDetectionOutput {
case progress(Double)
case result(String)
}
func detectTextFromPicture() {
SDWebImageManager.shared.loadImage(
with: viewModel.attachment.url,
options: [],
progress: nil) { image, _, _, _, _, _ in
guard let cgImage = image?.cgImage else { return }
self.detectText(cgImage: cgImage)
.sink { [weak self] in
if case let .failure(error) = $0 {
self?.present(alertItem: .init(error: error))
}
} receiveValue: { [weak self] in
guard let self = self else { return }
switch $0 {
case let .progress(progress):
self.detectTextFromPictureButton.isHidden = true
self.detectTextFromPictureProgressView.isHidden = false
self.detectTextFromPictureProgressView.progress = Float(progress)
case let .result(result):
self.detectTextFromPictureButton.isHidden = false
self.detectTextFromPictureProgressView.isHidden = true
self.textView.text += result
self.textViewDidChange(self.textView)
}
}
.store(in: &self.cancellables)
}
}
func detectText(cgImage: CGImage) -> AnyPublisher<TextDetectionOutput, Error> {
let subject = PassthroughSubject<TextDetectionOutput, Error>()
let recognizeTextRequest = VNRecognizeTextRequest { request, error in
if let error = error {
DispatchQueue.main.async {
subject.send(completion: .failure(error))
}
return
}
let recognizedTextObservations = request.results as? [VNRecognizedTextObservation] ?? []
let result = recognizedTextObservations
.compactMap { $0.topCandidates(1).first?.string }
.joined(separator: " ")
DispatchQueue.main.async {
subject.send(.result(result))
subject.send(completion: .finished)
}
}
recognizeTextRequest.recognitionLevel = .accurate
recognizeTextRequest.usesLanguageCorrection = true
recognizeTextRequest.progressHandler = { _, progress, error in
DispatchQueue.main.async {
if let error = error {
subject.send(completion: .failure(error))
return
}
subject.send(.progress(progress))
}
}
let handler = VNImageRequestHandler(cgImage: cgImage, options: [:])
do {
try handler.perform([recognizeTextRequest])
} catch {
subject.send(completion: .failure(error))
}
return subject.eraseToAnyPublisher()
}
}