[{"data":1,"prerenderedAt":295},["ShallowReactive",2],{"project-39113972":3},{"id":4,"name":5,"line":6,"line_source":7,"title":8,"url":9,"domain":10,"maker":11,"posted_at":12,"year":13,"points":14,"comments":15,"rank_in_year":16,"shown_in_year":17,"top":18,"tier":19,"category":20,"subcategory":21,"format":22,"keywords":23,"reception":29,"sentiment":30,"stance_mix":31,"scored_comments":35,"aspects":36,"strip":43,"post_text":45,"what":46,"said":65,"maker_in_thread":171,"shape":173,"grid":178,"attempts":256,"same_week":257,"maker_shows":40,"maker_hidden":287,"reader_version":288,"card":289},"39113972","nlm-ingestor","Open-source Rule-based PDF parser for RAG","title","Show HN: Open-source Rule-based PDF parser for RAG","https:\u002F\u002Fgithub.com\u002Fnlmatics\u002Fnlm-ingestor","github.com","jnathsf",1706074271,2024,293,32,214,3515,7,"legend","data","pdf parsing for rag","library",[24,25,26,27,28],"pdf parser","rag","ocr","document layout","chunking","mixed",0.202,{"positive":32,"neutral":32,"mixed":33,"critical":34},9,2,3,23,{"execution":37,"docs":38,"performance":41,"idea":42},{"positive":33,"critical":33},{"positive":39,"critical":40},0,1,{"positive":39,"critical":39},{"positive":33,"critical":39},[34,44,39,39,39,44,39,44,40,34,44,39,39,44,34,44,34,34,34,34,44,39,39,34,40,44,34,40,34,44,34,39],5,"The PDF parser is a rule based parser which uses text co-ordinates (boundary box), graphics and font data. The PDF parser works off text layer and also offers a OCR option to automatically use OCR if there are scanned pages in your PDFs. The OCR feature is based off a modified version of tika which uses tesseract underneath.\n\nThe PDF Parser offers the following features:\n\n* Sections and subsections along with their levels. * Paragraphs - combines lines. * Links between sections and paragraphs. * Tables along with the section the tables are found in. * Lists and nested lists. * Join content spr",{"problem":47,"audience":53,"pricing":53,"motivation":53,"time_to_build":53,"stack":54,"openness":63,"stage":64,"team_size":64},{"value":48,"source":49,"evidence":50},"Extracting structured text from PDFs for RAG pipelines","maker",[51],{"id":52},"post",null,[55,59],{"value":56,"source":49,"evidence":57},"Tika",[58],{"id":52},{"value":60,"source":49,"evidence":61},"Tesseract",[62],{"id":52},"open_source","unknown",{"praise":66,"objections":86,"requests":111,"questions":124,"alternatives":139},[67,75],{"value":68,"source":69,"evidence":70},"Chunking keeps context together and table parsing is good","commenter",[71],{"id":72,"by":73,"text":74},"39119201","mpeg","Thanks, I always forget I can do that! I've given it a go and it's really impressive – the default chunker is very smart and manages to keep most of the chunk context together…",{"value":76,"source":69,"evidence":77},"Fills a need for better RAG document loading",[78,82],{"id":79,"by":80,"text":81},"39118145","xfalcox","We've been looking for something exactly like this, thanks for sharing!",{"id":83,"by":84,"text":85},"39117354","rmsaksida","Last time I tried Langchain (admittedly, that was ~6 months ago) the implementations for content extraction from PDFs and HTML files were very basic. Enough to get a prototype…",[87,93,104],{"value":88,"source":69,"evidence":89},"Docker image does not build natively on Apple silicon",[90],{"id":91,"by":73,"text":92},"39116662","I couldn't try this tool as it doesn't build on apple silicon (and there's no ARM docker image) However, I have a PDF parsing use-case that I tried those RAG tools for, but the…",{"value":94,"source":69,"evidence":95},"Hard to evaluate without example PDFs or input\u002Foutput pairs in the repo",[96,100],{"id":97,"by":98,"text":99},"39114766","jvdvegt","Do you ave any examples? There doesn't seem to be a single PDF file in the repo.",{"id":101,"by":102,"text":103},"39115244","firtoz","Thank you for sharing. Are there some example input output pairs somewhere?",{"value":105,"source":69,"evidence":106},"Tesseract OCR makes mistakes",[107],{"id":108,"by":109,"text":110},"39116617","cdolan","I am also curious about this. ADI is reliable but does have edge case issues on malformed PDF I fear tesseract OCR is a potential limitation though. I’ve seen it make so many…",[112,117],{"value":113,"source":69,"evidence":114},"Provide example inputs and outputs",[115,116],{"id":101,"by":102,"text":103},{"id":97,"by":98,"text":99},{"value":118,"source":69,"evidence":119},"Benchmark comparison against other loaders",[120],{"id":121,"by":122,"text":123},"39116537","lmeyerov","Tesseract OCR fallback sounds great! There are now a lot of file loaders for RAG (langchain, LLMindex, unstructured, ...), any reasons, like a leading benchmark score, to prefer…",[125,132],{"value":126,"source":69,"evidence":127},"How does it compare to Azure Document Intelligence",[128],{"id":129,"by":130,"text":131},"39114407","guidedlight","How does this differ from Azure Document Intelligence, or are they effectively the same thing?",{"value":133,"source":69,"evidence":134},"How does it compare to PaddleOCR",[135],{"id":136,"by":137,"text":138},"39117167","ilaksh","How does this compare to PaddleOCR? Looks like Apache 2 license which is nice.",[140,147,151,158,164],{"value":141,"source":69,"evidence":142},"llmsherpa",[143],{"id":144,"by":145,"text":146},"39117586","asukla","Thanks for the post. Please use this server with the llmsherpa LayoutPDFReader to get optimal chunks for your LLM\u002FRAG project: https:\u002F\u002Fgithub.com\u002Fnlmatics\u002Fllmsherpa. See examples…",{"value":148,"source":69,"evidence":149},"Azure Document Intelligence",[150],{"id":129,"by":130,"text":131},{"value":152,"source":69,"evidence":153},"PyMuPDF\u002Ffitz",[154],{"id":155,"by":156,"text":157},"39117465","ramoz","For me, PyMuPDF\u002Ffitz has been the best way to retain natural reading order and set dynamic enough rules to extract text in complex layouts. None of the mentioned tools did this…",{"value":159,"source":69,"evidence":160},"pdfminer.six",[161],{"id":162,"by":73,"text":163},"39116723","Off-topic, but do you know how Tika compares to other pdf parsing libraries? I was very unimpressed by pdfminer.six (what unstructured uses) as the layout detection seems pretty…",{"value":165,"source":69,"evidence":166},"Tabula",[167],{"id":168,"by":169,"text":170},"39119971","jahewson","Tika uses PDFBox under the hood, using its built-in text extractor (which is \"ok\"). If you're looking for table extraction specifically, check out Tabula…",{"comments":39,"answered":39,"key":172},[],{"commenters":174,"max_depth":175,"maker_replies":39,"first_comment_mins":176,"lifespan_hours":177},21,4,80,38,[179,183,186,188,190,191,194,198,199,201,204,206,207,209,210,214,215,218,220,224,227,231,234,237,239,240,242,244,246,248,250,252],[180,39,34,181,182,39],"39116858","project\u002Fneutral\u002Fnone","dmezzetti",[184,40,44,185,182,39],"39116082","project\u002Fpositive\u002Fnone",[162,40,39,187,73,39],"other\u002Fneutral\u002Fnone",[189,40,39,187,182,39],"39116846",[168,40,39,187,169,39],[192,33,44,185,193,39],"39115202","epaga",[195,33,39,196,197,39],"39134020","commenter\u002Fneutral\u002Fnone","muzamil-ali",[121,34,44,185,122,39],[91,34,40,200,73,39],"project\u002Fcritical\u002Fexecution",[202,34,34,203,145,39],"39117452","project\u002Fneutral\u002Fexecution",[72,34,44,205,73,39],"project\u002Fpositive\u002Fexecution",[155,34,39,187,156,39],[208,34,39,187,73,39],"39118796",[83,34,44,205,84,39],[211,175,34,212,213,39],"39119372","maker\u002Fmixed\u002Fnone","mistrial9",[144,44,44,185,145,39],[101,216,34,217,102,39],6,"project\u002Fneutral\u002Fdocs",[219,216,34,217,145,39],"39117471",[221,18,34,222,223,39],"39118547","project\u002Fmixed\u002Fperformance","huqedato",[129,225,34,226,130,39],8,"project\u002Fneutral\u002Fidea",[228,225,44,229,230,39],"39114599","project\u002Fpositive\u002Fidea","StrauXX",[232,225,39,187,233,39],"39116778","infecto",[235,225,39,187,236,39],"39116853","batch12",[238,225,34,217,145,39],"39117807",[108,225,40,200,109,39],[241,225,44,229,145,39],"39117484",[243,225,34,226,156,39],"39117508",[97,32,40,245,98,39],"project\u002Fcritical\u002Fdocs",[247,32,34,217,145,39],"39117515",[79,249,44,185,80,39],10,[136,251,34,181,137,39],11,[253,254,39,187,255,39],"39127256",12,"genewitch",[],[258,263,268,273,278,282],{"id":259,"name":260,"points":261,"category":262},"39120647","funding simulator",654,"finance_crypto",{"id":264,"name":265,"points":266,"category":267},"39113879","teachyourselfmath",384,"learning_science",{"id":269,"name":270,"points":271,"category":272},"39067496","Coffeehouse",229,"consumer_social",{"id":274,"name":275,"points":276,"category":277},"39105502","Retriever",193,"security_privacy",{"id":279,"name":280,"points":281,"category":267},"39072079","Checkmate Champ",192,{"id":283,"name":284,"points":285,"category":286},"39099980","Kener",187,"infrastructure",false,"v2",{"id":290,"name":5,"line":6,"ai":287,"cat":20,"year":13,"points":14,"comments":15,"rec":29,"s":291,"top":18,"kw":292,"maker":11,"domain":10,"tier":19,"attempts":39,"strip":293,"url":9,"mix":294,"praise":68,"objection":88,"sub":21},39113972,0.2,[24,25,26,27,28],"35000505135005353333500315313530",[32,32,33,34],1794036648641]