[{"data":1,"prerenderedAt":340},["ShallowReactive",2],{"project-43118514":3},{"id":4,"name":5,"line":6,"line_source":7,"title":8,"url":9,"domain":10,"maker":11,"posted_at":12,"year":13,"points":14,"comments":15,"rank_in_year":16,"shown_in_year":17,"top":18,"tier":19,"category":20,"subcategory":21,"format":22,"keywords":23,"reception":27,"sentiment":28,"stance_mix":29,"scored_comments":33,"aspects":34,"strip":42,"post_text":45,"what":46,"said":62,"maker_in_thread":190,"shape":193,"grid":197,"attempts":302,"same_week":303,"maker_shows":273,"maker_hidden":332,"reader_version":333,"card":334},"43118514","OmniAI OCR Benchmark","Benchmarking VLMs vs. Traditional OCR","title","Show HN: Benchmarking VLMs vs. Traditional OCR","https:\u002F\u002Fgetomni.ai\u002Focr-benchmark","getomni.ai","themanmaran",1740077369,2025,146,40,510,3837,14,"notable","data","ocr benchmark","other",[21,24,25,26],"vision language models","ocr accuracy","document extraction","curious",-0.205,{"positive":30,"neutral":31,"mixed":30,"critical":32},1,12,9,23,{"idea":35,"performance":37,"execution":39,"privacy":41},{"positive":36,"critical":30},0,{"positive":30,"critical":38},2,{"positive":36,"critical":40},4,{"positive":36,"critical":38},[43,43,43,36,36,36,30,30,36,43,36,30,44,36,30,30,36,36,36,36,43,43,43,30,36,30,36,36,43,36,43,30,43,43,36,36,30,43,36,43],3,5,"Vision models have been gaining popularity as a replacement for traditional OCR. Especially with Gemini 2.0 becoming cost competitive with the cloud platforms.\n\nWe've been continuously evaluating different models since we released the Zerox package last year (https:\u002F\u002Fgithub.com\u002Fgetomni-ai\u002Fzerox). And we wanted to put some numbers behind it. So we’re open sourcing our internal OCR benchmark + evaluation datasets.\n\nFull writeup + data explorer here: https:\u002F\u002Fgetomni.ai\u002Focr-benchmark\n\nGithub: https:\u002F\u002Fgithub.com\u002Fgetomni-ai\u002Fbenchmark\n\nHuggingface: https:\u002F\u002Fhuggingface.co\u002Fdatasets\u002Fgetomni-ai\u002Focr-bench",{"problem":47,"audience":53,"pricing":53,"motivation":54,"time_to_build":53,"stack":58,"openness":59,"stage":60,"team_size":61},{"value":48,"source":49,"evidence":50},"Lack of numbers comparing VLM and traditional OCR accuracy for LLM-ready data extraction","maker",[51],{"id":52},"post",null,{"value":55,"source":49,"evidence":56},"Put numbers behind their continuous evaluation of OCR models since releasing Zerox",[57],{"id":52},[],"open_source","launched","company",{"praise":63,"objections":80,"requests":109,"questions":143,"alternatives":169},[64,72],{"value":65,"source":66,"evidence":67},"Honest, open data release from OmniAI","commenter",[68],{"id":69,"by":70,"text":71},"43155742","blindriver","As far as I'm concerned, all of these specialty services are dead compared to a generalized LLM like OpenAI or Gemini. I wrote in a previous post about how NLP services were dead…",{"value":73,"source":66,"evidence":74},"Gemini Flash 2.0 is a strong, cheap OCR option",[75,79],{"id":76,"by":77,"text":78},"43153303","jasonjmcghee","A big takeaway for me is that Gemini Flash 2.0 is a great solution to OCR, considering accessibility, cost, accuracy, and speed. It also has a 1M token context window, though…",{"id":69,"by":70,"text":71},[81,92,102],{"value":82,"source":66,"evidence":83},"GPT-4o as judge and structured extraction task is not a true OCR benchmark",[84,88],{"id":85,"by":86,"text":87},"43158548","anon373839","I’m definitely not getting that takeaway. This wasn’t even an OCR benchmark: the task was structured data extraction, and deterministic metrics were set aside in favor of…",{"id":89,"by":90,"text":91},"43161193","danielcampos93","GPT-4o as a judge to evaluate the quality of something which gpt4o is not inherently that good at. Red flag.",{"value":93,"source":66,"evidence":94},"Accuracy of 87-92% is still too error-prone for numbers and money",[95,99],{"id":96,"by":97,"text":98},"43160436","toss1","YES >>5% accuracy can be worth a lot. Most surprising to me about these results is the BEST error rate was over 8% errors (91.7% accuracy) and the worse was 40%. Their method of…",{"id":100,"by":70,"text":101},"43161956","I think 87% to 92% accuracy really isn't much difference. You're still going to get errors to the point where the level and amount of checking you need to do isn't affected. Even…",{"value":103,"source":66,"evidence":104},"VLMs can hallucinate and correct source documents, unsuitable for important OCR",[105,106],{"id":85,"by":86,"text":87},{"id":107,"by":11,"text":108},"43164117","We've tested basic prompt injections within images, but not been able to reliably trigger any adverse effects. However there are two big bugs we've found with VLMs: 1. Correcting…",[110,129,136],{"value":111,"source":66,"evidence":112},"Add open models like Qwen 2.5 VL and small VLMs",[113,117,121,125],{"id":114,"by":115,"text":116},"43131719","betula_ai","Thank you for sharing this. Some of the other public models that we can host ourselves may perform in practice better than the models listed - e.g. Qwen 2.5 VL…",{"id":118,"by":119,"text":120},"43154968","banditelol","Anyone have tried comparing with Qwen VL based model? I heard good things about its performance on ocr compared to other self hostable model, but haven't really tried…",{"id":122,"by":123,"text":124},"43157990","jimmySixDOF","Yes I'd like to see this repeated with any of the small VLM's like IBM Granite or the HF Smols. Pretty much anything in the sub 7B range.",{"id":126,"by":127,"text":128},"43192541","default_","Wondering why Florence-2 is not on the list of models?",{"value":130,"source":66,"evidence":131},"Add free engines like Tesseract for comparison",[132],{"id":133,"by":134,"text":135},"43159839","bayindirh","Also, what's strange is there's no free of paid OCR engine is added to the mix for the evaluation. Tessaract is built specially for OCR'in scanned documents, and it has a both…",{"value":137,"source":66,"evidence":138},"Benchmark prompt injection risks in OCR",[139],{"id":140,"by":141,"text":142},"43154799","simonw","The benchmark I most want to see around OCR is one that covers risks from accidental (or deliberate) prompt injection - I want to know how likely it is that a model might OCR a…",[144,151,158],{"value":145,"source":66,"evidence":146},"What is the privacy of documents sent to the cloud API?",[147],{"id":148,"by":149,"text":150},"43154387","bn-l","What is the privacy of the documents for the cloud service? There’s nothing in the privacy policy about data sent over the api.",{"value":152,"source":66,"evidence":153},"How does this compare to Marker?",[154],{"id":155,"by":156,"text":157},"43154779","gbertb","How does this compare to Marker https:\u002F\u002Fgithub.com\u002FVikParuchuri\u002Fmarker?",{"value":159,"source":66,"evidence":160},"Which VLMs are used in OmniAI?",[161,165],{"id":162,"by":163,"text":164},"43192538","hasibzunair","What kind of VLMs are being used in OmniAI? I fine-tuned a Llama 3.2 Vision on a small dataset I created for extracting text without heavy cropping. Results are simply amazing in…",{"id":166,"by":167,"text":168},"43120771","fzysingularity","What VLMs do you use when you're listing OmniAI - is this mostly wrapping the model providers like your zerox repo?",[170,175,179,186],{"value":171,"source":66,"evidence":172},"Google Gemini",[173,174],{"id":69,"by":70,"text":71},{"id":76,"by":77,"text":78},{"value":176,"source":66,"evidence":177},"Tesseract",[178],{"id":133,"by":134,"text":135},{"value":180,"source":66,"evidence":181},"ABBYY FineReader",[182],{"id":183,"by":184,"text":185},"43160784","sgc","> What would be interesting is finding which two OCR\u002FAI systems make the most different mistakes and running documents against both. Flagging only the disagreements for human…",{"value":187,"source":66,"evidence":188},"Marker",[189],{"id":155,"by":156,"text":157},{"comments":30,"answered":36,"key":191},[192],{"id":107,"text":108},{"commenters":194,"max_depth":44,"maker_replies":30,"first_comment_mins":195,"lifespan_hours":196},36,184,158,[198,200,203,207,209,213,217,219,223,227,229,230,231,233,237,239,240,243,245,247,250,251,254,256,260,263,266,269,272,274,278,282,285,286,288,290,294,295,297,300],[140,36,43,199,141,36],"project\u002Fneutral\u002Fidea",[201,36,43,199,202,36],"43158482","mentalgear",[204,36,43,205,206,36],"43162341","project\u002Fneutral\u002Fnone","magicalhippo",[107,36,36,208,11,30],"maker\u002Fneutral\u002Fperformance",[210,36,36,211,212,36],"43166839","other\u002Fneutral\u002Fnone","th0ma5",[214,36,36,215,216,36],"43172894","other\u002Fneutral\u002Fidea","falcor84",[69,30,30,218,70,36],"project\u002Fcritical\u002Fidea",[220,30,30,221,222,36],"43158700","project\u002Fcritical\u002Fperformance","modo_mario",[224,30,36,225,226,36],"43160080","commenter\u002Fneutral\u002Fnone","Eridrus",[96,30,43,228,97,36],"project\u002Fmixed\u002Fperformance",[183,30,36,225,184,36],[100,30,30,221,70,36],[76,38,44,232,77,36],"project\u002Fpositive\u002Fperformance",[234,38,36,235,236,36],"43153908","other\u002Fpositive\u002Fnone","shawabawa3",[85,38,30,238,86,36],"project\u002Fcritical\u002Fexecution",[133,38,30,238,134,36],[241,38,36,225,242,36],"43160705","michaelt",[244,38,36,225,134,36],"43171739",[246,38,36,225,242,36],"43160659",[248,38,36,211,249,36],"43161551","pzo",[155,43,43,205,156,36],[252,43,43,205,253,36],"43159710","marcotac",[148,40,43,255,149,36],"project\u002Fneutral\u002Fprivacy",[257,40,30,258,259,36],"43156922","project\u002Fcritical\u002Fprivacy","dcreater",[261,40,36,262,149,36],"43160664","other\u002Fneutral\u002Fprivacy",[264,40,30,258,265,36],"43158468","cess11",[267,44,36,211,268,36],"43133356","EarlyOom",[270,44,36,225,271,36],"43153998","codelion",[114,273,43,199,115,36],6,[275,276,36,211,277,36],"43160802",7,"alkh",[279,280,43,199,281,36],"43156354",8,"alok-g",[283,280,30,238,284,36],"43159195","lyu07282",[118,32,43,205,119,36],[122,32,43,287,123,36],"project\u002Fneutral\u002Fperformance",[162,289,36,235,163,36],10,[291,292,36,211,293,36],"43157691",11,"westurner",[89,31,30,238,90,36],[166,296,43,205,167,36],13,[298,18,36,211,299,36],"43160964","jll29",[126,301,43,205,127,36],15,[],[304,309,314,319,324,328],{"id":305,"name":306,"points":307,"category":308},"43186406","Ballionaire",1581,"games",{"id":310,"name":311,"points":312,"category":313},"43158660","touchgrass.now",1203,"productivity",{"id":315,"name":316,"points":317,"category":318},"43120582","Sutro Tower Gaussian Splat",825,"creative",{"id":320,"name":321,"points":322,"category":323},"43121383","BadSeek",461,"ai",{"id":325,"name":326,"points":327,"category":308},"43183131","Breakout (roguelite)",412,{"id":329,"name":330,"points":331,"category":313},"43162340","corporate",392,false,"v2",{"id":335,"name":5,"line":6,"ai":332,"cat":20,"year":13,"points":14,"comments":15,"rec":27,"s":336,"top":18,"kw":337,"maker":11,"domain":10,"tier":19,"attempts":36,"strip":338,"url":9,"mix":339,"praise":65,"objection":82,"sub":21},43118514,-0.2,[21,24,25,26],"3330001103015011000033310100303133001303",[30,31,30,32],1794036670022]