Find the binary
Match any likely screen ai related names
Found: ~/Library/Application Support/Google/Chrome/screen_ai/148.7/libchromescreenai.so, about 50 MB
find ~/Library/Application\ Support/Google/Chrome -iname '*screen*ai*' -o -iname '*screen_ai*' -o -iname '*screenai*' 2>/dev/null | head -40
Analyze the binary
If you have not installed rg (ripgrep), install now.
brew install ripgrep
Now try to read the symbols
# -g: show only external/global symbols
# -U: omit undefined symbols, so you only see symbols defined by this binary
# T: symbol is in the text/code section
nm -gU ~/Library/Application\ Support/Google/Chrome/screen_ai/148.7/libchromescreenai.so 2>/dev/null | head -80
Found:
0000000100013dd4 T _EnableDebugMode
0000000100013d84 T _ExtractMainContent
0000000100013dc8 T _FreeLibraryAllocatedCharArray
0000000100013dbc T _FreeLibraryAllocatedInt32Array
0000000100013af8 T _GetLibraryVersion
0000000100013d1c T _GetMaxImageDimension
0000000100013d64 T _InitMainContentExtractionUsingCallback
0000000100013d0c T _InitOCRUsingCallback
0000000100013d3c T _PerformOCR
0000000100013ce8 T _SetFileContentFunctions
0000000100013cec T _SetOCRLightMode
0000000100013d74 T _UninitializeMainContentExtraction
0000000100013d2c T _UninitializeOCR
From https://chromium.googlesource.com/chromium/src/+/HEAD/services/screen_ai/
could see the directly related wrapper files are:
screen_ai_library_wrapper.h
screen_ai_library_wrapper_fake.cc
screen_ai_library_wrapper_fake.h
screen_ai_library_wrapper_impl.cc
screen_ai_library_wrapper_impl.h
screen_ai_ocr_perf_test.cc
screen_ai_service_impl.cc
screen_ai_service_impl.h
screen_ai_service_impl_unittest.cc
based on screen_ai_library_wrapper.h, could see tha InitOCR() runs first, then PerformOcr(const SkBitmap&) -> VisualAnnotation
The function all format of other related calls are also in screen_ai_library_wrapper.h.
however, we don’t know how SkBitmap looks like.
now figure out what fields are needed for SkBitmap:
# deassembly
# search for names
# show prev/after 30-line context
otool -tvV ~/Library/Application\ Support/Google/Chrome/screen_ai/148.7/libchromescreenai.so | rg -n 'chrome_screen_ai::OCR::PerformOCR|GetPngString|PerformOCR' -C 30
ARM64 calling convention passes the first four arguments in x0, x1, x2, x3.
19590:_PerformOCR:
19591-0000000100013d3c stp x20, x19, [sp, #-0x20]!
19592-0000000100013d40 stp x29, x30, [sp, #0x10]
19593-0000000100013d44 mov x19, x1 ; output length pointer
19594-0000000100013d48 mov x20, x0 ; SkBitmap const& argument
19595-0000000100013d4c bl __ZN16chrome_screen_ai3OCR11GetInstanceEv
19596-0000000100013d50 mov x1, x20 ; arg1 = SkBitmap*
19597-0000000100013d54 mov x2, x19 ; arg2 = length*
19598-0000000100013d58 ldp x29, x30, [sp, #0x10]
19599-0000000100013d5c ldp x20, x19, [sp], #0x20
19600:0000000100013d60 b __ZN16chrome_screen_ai3OCR10PerformOCRERK8SkBitmapRj
closer look at OCR::PerformOCR
794863:__ZN16chrome_screen_ai3OCR10PerformOCRERK8SkBitmapRj:
794864-00000001002ff004 sub sp, sp, #0x1f0
794865-00000001002ff008 stp x28, x27, [sp, #0x1b0]
794866-00000001002ff00c stp x22, x21, [sp, #0x1c0]
794867-00000001002ff010 stp x20, x19, [sp, #0x1d0]
794868-00000001002ff014 stp x29, x30, [sp, #0x1e0]
794869-00000001002ff018 mov x19, x2
794870-00000001002ff01c mov x21, x1 ; x21 = SkBitmap const*
794871-00000001002ff020 mov x20, x0 ; x20 = this
794872-00000001002ff024 ldr x8, [x0, #0x8]
794873-00000001002ff028 cbnz x8, 0x1002ff038
794874-00000001002ff02c mov x0, x20 ; x0 = this
794875-00000001002ff030 bl __ZN16chrome_screen_ai3OCR10InitializeEv ; call OCR::Initialize()
794876-00000001002ff034 cbz w0, 0x1002ff188
794877-00000001002ff038 bl __ZN16chrome_screen_ai11IsDebugModeEv ; call IsDebugMode()
794878-00000001002ff03c cbz w0, 0x1002ff084
794879-00000001002ff040 add x22, sp, #0x60
794880-00000001002ff044 add x0, sp, #0x60
794881-00000001002ff048 mov x1, x21 ; x1 = SkBitmap const*
794882:00000001002ff04c bl __ZN12_GLOBAL__N_112GetPngStringERK8SkBitmap ; call GetPngString(SkBitmap const&)
794883-00000001002ff050 ldrb w8, [sp, #0x77]
794884-00000001002ff054 sxtb w9, w8
794885-00000001002ff058 ldp x10, x11, [sp, #0x60]
794886-00000001002ff05c cmp w9, #0x0
794887-00000001002ff060 csel x0, x10, x22, mi
794888-00000001002ff064 csel x1, x11, x8, mi
794889-00000001002ff068 adrp x2, 7373 ; 0x101fcc000
794890-00000001002ff06c add x2, x2, #0xcfc ; literal pool for: "visual_annotations_input"
794891-00000001002ff070 adrp x3, 7373 ; 0x101fcc000
794892-00000001002ff074 add x3, x3, #0xb53 ; literal pool for: "png"
794893-00000001002ff078 bl __ZN16chrome_screen_ai14WriteDebugFileENSt3__117basic_string_viewIcNS0_11char_traitsIcEEEEPKcS6_
794894-00000001002ff07c ldrsb w8, [sp, #0x77]
794895-00000001002ff080 tbnz w8, #0x1f, 0x1002ff1a4
794896-00000001002ff084 ldr w9, [x21, #0x28] ; likely SkBitmap width
794897-00000001002ff088 cbz w9, 0x1002ff1bc
794898-00000001002ff08c ldr w8, [x21, #0x2c] ; likely SkBitmap height
794899-00000001002ff090 cmp w8, #0x8, lsl #12
794900-00000001002ff094 b.gt 0x1002ff1bc
794901-00000001002ff098 cmp w9, #0x8, lsl #12
794902-00000001002ff09c b.gt 0x1002ff1bc
794903-00000001002ff0a0 cbz w8, 0x1002ff1bc
794904-00000001002ff0a4 mov w5, #0x0
794905-00000001002ff0a8 ldr w8, [x21, #0x20] ; likely SkBitmap SkColorType
794906-00000001002ff0ac cmp w8, #0x4 ; kRGBA_8888_SkColorType
794907-00000001002ff0b0 b.eq 0x1002ff0c8
794908-00000001002ff0b4 cmp w8, #0x6 ; kBGRA_8888_SkColorType
794909-00000001002ff0b8 b.eq 0x1002ff0c8
794910-00000001002ff0bc cmp w8, #0xe ; kGray_8_SkColorType
794911-00000001002ff0c0 b.ne 0x1002ff1f4
794912-00000001002ff0c4 mov w5, #0x3
Then the structure of the SkBitmap passed to PerformOCR is:
struct FakeSkBitmap {
// Private Skia ABI shape observed from libchromescreenai.so 148.7 on arm64.
// This is not a public API.
uint64_t pad0 = 0;
const uint8_t* pixels = nullptr; // offset 0x08
uint64_t pad1 = 0;
uint64_t pad2 = 0;
int32_t color_type = 4; // offset 0x20, kRGBA_8888_SkColorType
int32_t alpha_type = 0;
int32_t width = 0; // offset 0x28
int32_t height = 0; // offset 0x2c
uint8_t rest[128] = {};
};
Call the OCR function
// create type aliases for function pointers
//SetFileContentFunctions(&GetFileContentSize, &GetFileContent);
// uint32_t GetFileContentSize(const char* path); // → 1st arg
// void GetFileContent(const char*, uint32_t, char*); // → 2nd arg
using SetFileContentFunctionsFn =
void (*)(uint32_t (*)(const char*),
void (*)(const char*, uint32_t, char*));
// bool InitOCRUsingCallback();
using InitOCRFn = bool (*)();
// char* PerformOCR(const SkBitmap& bitmap, uint32_t& out_length);
using PerformOCRFn = char* (*)(const FakeSkBitmap&, uint32_t&);
// void FreeLibraryAllocatedCharArray(char* ptr);
using FreeCharArrayFn = void (*)(char*);
// void UninitializeOCR();
using UninitializeOCRFn = void (*)();
auto set_files =
LoadSymbol<SetFileContentFunctionsFn>(lib, "SetFileContentFunctions");
auto init_ocr = LoadSymbol<InitOCRFn>(lib, "InitOCRUsingCallback");
auto perform_ocr = LoadSymbol<PerformOCRFn>(lib, "PerformOCR");
auto free_chars =
LoadSymbol<FreeCharArrayFn>(lib, "FreeLibraryAllocatedCharArray");
auto uninit_ocr = LoadSymbol<UninitializeOCRFn>(lib, "UninitializeOCR");
set_files(&GetFileContentSize, &GetFileContent);
if (!init_ocr()) {
std::cerr << "InitOCRUsingCallback failed\n";
return 3;
}
FakeSkBitmap bitmap;
bitmap.pixels = pixels.data();
bitmap.width = width;
bitmap.height = height;
uint32_t result_len = 0;
char* result = perform_ocr(bitmap, result_len);