PubHarbor changes to libmspub 0.1.5 (MPL-2.0; modified files keep their MPL header): - Page number fields: parse the Quill TOKN chunks, mark each page-number token's '#' placeholder, and print the number of the page being drawn (master-page footers showed a literal '#'). - Publisher 97-2000: skip the hidden page that holds one default object per shape type (it rendered as an empty first or last page); detect it structurally instead of relying on fixed sequence numbers only. - Publisher 2000 tables (content chunk type 0x000A): read the column widths, row heights, cell spans and the cell text story so pages whose only object is a table render instead of failing with "no renderable pages". - Publisher 97/98 text code page: use the code page implied by a script-specific font family in the font table ("... Cyr", "... CE", "... Greek", "... Tur", "... Baltic") before ICU's statistical detection, which misreads short Cyrillic text as windows-1252. diff -ruN s1/src/lib/MSPUBCollector.cpp cur/src/lib/MSPUBCollector.cpp --- a/src/lib/MSPUBCollector.cpp 2026-09-25 18:40:32 +++ b/src/lib/MSPUBCollector.cpp 2026-09-25 18:40:32 @@ -11,6 +11,7 @@ #include #include +#include #include #include @@ -83,6 +84,29 @@ iface->insertText(tmpText); } +// PubHarbor: MSPUBParser marks a page number field with PAGE_NUMBER_FIELD_CHAR +// (U+E1A0 from the private use area) in place of its '#' placeholder. Show the +// number of the page being drawn, as Publisher does. +static librevenge::RVNGString fillPageNumberFields(const librevenge::RVNGString &text, unsigned pageNumber) +{ + static const char marker[] = "\xEE\x86\xA0"; + const std::string source(text.cstr()); + if (source.find(marker) == std::string::npos) + return text; + const std::string number = pageNumber ? std::to_string(pageNumber) : std::string("#"); + std::string result; + size_t cursor = 0; + size_t found; + while ((found = source.find(marker, cursor)) != std::string::npos) + { + result.append(source, cursor, found - cursor); + result += number; + cursor = found + sizeof(marker) - 1; + } + result.append(source, cursor, std::string::npos); + return librevenge::RVNGString(result.c_str()); +} + static void separateSpacesAndInsertText(librevenge::RVNGDrawingInterface *iface, const librevenge::RVNGString &text) { if (!iface) @@ -494,7 +518,7 @@ m_tableCellTextEndsByTextId(), m_stringOffsetsByTextId(), m_calculationValuesSeen(), m_pageSeqNumsOrdered(), m_encodingHeuristic(false), m_allText(), - m_calculatedEncoding(), + m_calculatedEncoding(), m_fallbackEncoding(), m_currentPageNumber(0), m_metaData() { } @@ -511,6 +535,11 @@ m_tableCellTextEndsByTextId[textId] = ends; } +void MSPUBCollector::setFallbackEncoding(const char *const encoding) +{ + m_fallbackEncoding = encoding; +} + void MSPUBCollector::useEncodingHeuristic() { m_encodingHeuristic = true; @@ -1169,7 +1198,7 @@ { librevenge::RVNGPropertyList charProps = getCharStyleProps(text[para].spans[i_spans].style, text[para].style.m_defaultCharStyleIndex); m_painter->openSpan(charProps); - separateSpacesAndInsertText(m_painter, paraTexts[para][i_spans]); + separateSpacesAndInsertText(m_painter, fillPageNumberFields(paraTexts[para][i_spans], m_currentPageNumber)); m_painter->closeSpan(); } @@ -1233,7 +1262,7 @@ getCalculatedEncoding()); librevenge::RVNGPropertyList charProps = getCharStyleProps(line.spans[i_spans].style, line.style.m_defaultCharStyleIndex); m_painter->openSpan(charProps); - separateSpacesAndInsertText(m_painter, textString); + separateSpacesAndInsertText(m_painter, fillPageNumberFields(textString, m_currentPageNumber)); m_painter->closeSpan(); } m_painter->closeParagraph(); @@ -1260,6 +1289,15 @@ m_calculatedEncoding = "UTF-16LE"; return m_calculatedEncoding.get(); } + // PubHarbor: the parser found the file's code page (for example from a + // script-specific font name); trust it over statistical detection, which is + // unreliable on short publication strings. + if (bool(m_fallbackEncoding) && + std::any_of(m_allText.begin(), m_allText.end(), [](unsigned char c) { return c >= 0x80; })) + { + m_calculatedEncoding = m_fallbackEncoding.get(); + return m_calculatedEncoding.get(); + } // for older versions of PUB, see if we can get ICU to tell us the encoding. UErrorCode status = U_ZERO_ERROR; UCharsetDetector *ucd = nullptr; @@ -1308,7 +1346,7 @@ csd_fail: ucsdet_close(ucd); // Pretty likely to give garbage text, but it's the best we can do. - m_calculatedEncoding = "windows-1252"; + m_calculatedEncoding = bool(m_fallbackEncoding) ? m_fallbackEncoding.get() : "windows-1252"; return m_calculatedEncoding.get(); } @@ -1783,6 +1821,7 @@ const auto &shapeGroupsOrdered = pageInfo.m_shapeGroupsOrdered; if (!shapeGroupsOrdered.empty()) { + ++m_currentPageNumber; m_painter->startPage(pageProps); boost::optional masterSeqNum = getMasterPageSeqNum(pageSeqNum); auto hasMaster = bool(masterSeqNum); diff -ruN s1/src/lib/MSPUBCollector.h cur/src/lib/MSPUBCollector.h --- a/src/lib/MSPUBCollector.h 2026-09-25 18:40:32 +++ b/src/lib/MSPUBCollector.h 2026-09-25 18:40:32 @@ -120,6 +120,7 @@ bool setCurrentGroupSeqNum(unsigned seqNum); void useEncodingHeuristic(); + void setFallbackEncoding(const char *encoding); void setTableCellTextEnds(unsigned textId, const std::vector &ends); void setTextStringOffset(unsigned textId, unsigned offset); @@ -172,6 +173,8 @@ bool m_encodingHeuristic; std::vector m_allText; mutable boost::optional m_calculatedEncoding; + boost::optional m_fallbackEncoding; + mutable unsigned m_currentPageNumber; librevenge::RVNGPropertyList m_metaData; // helper functions diff -ruN s1/src/lib/MSPUBConstants.h cur/src/lib/MSPUBConstants.h --- a/src/lib/MSPUBConstants.h 2026-09-25 18:40:32 +++ b/src/lib/MSPUBConstants.h 2026-09-25 18:40:32 @@ -13,6 +13,8 @@ #define EMUS_IN_INCH 914400 #define POINTS_IN_INCH 72 #define DEFAULT_MARGIN 36576 +// PubHarbor: private-use marker for a page number field in Quill text. +#define PAGE_NUMBER_FIELD_CHAR 0xE1A0u #endif /* INCLUDED_MSPUBCONSTANTS_H */ /* vim:set shiftwidth=2 softtabstop=2 expandtab: */ diff -ruN s1/src/lib/MSPUBParser.cpp cur/src/lib/MSPUBParser.cpp --- a/src/lib/MSPUBParser.cpp 2026-09-25 18:40:32 +++ b/src/lib/MSPUBParser.cpp 2026-09-25 18:40:31 @@ -1046,6 +1046,39 @@ return ret; } +// PubHarbor: a TOKN chunk is a PLC of field tokens for the story with the same +// index. Each entry is a length-prefixed list of (UINT16 key, UINT32 value) +// properties; key 0x2202 is the token type, and type 0xFFFFFFFB is a page number +// field whose '#' placeholder starts the token's range. +std::set MSPUBParser::parsePageNumberFields(librevenge::RVNGInputStream *input, const QuillChunkReference &chunk) +{ + std::set fields; + const unsigned long end = chunk.offset + chunk.length; + const unsigned numEntries = readU32(input); + if (numEntries == 0 || numEntries > 0x10000 || chunk.offset + 12 + 4ul * (numEntries + 1) > end) + return fields; + input->seek(chunk.offset + 12, librevenge::RVNG_SEEK_SET); + std::vector positions; + for (unsigned i = 0; i <= numEntries; ++i) + positions.push_back(readU32(input)); + for (unsigned i = 0; i < numEntries && stillReading(input, end); ++i) + { + const unsigned long entryStart = input->tell(); + const unsigned entryLength = readU32(input); + if (entryLength < 4 || entryStart + entryLength > end) + break; + while (input->tell() + 6 <= entryStart + entryLength) + { + const unsigned key = readU16(input); + const unsigned value = readU32(input); + if (key == 0x2202 && value == 0xFFFFFFFB) + fields.insert(positions[i]); + } + input->seek(entryStart + entryLength, librevenge::RVNG_SEEK_SET); + } + return fields; +} + bool MSPUBParser::parseQuill(librevenge::RVNGInputStream *input) { MSPUB_DEBUG_MSG(("MSPUBParser::parseQuill\n")); @@ -1082,6 +1115,7 @@ std::vector textIDs; std::vector textOffsets; std::map > tableCellTextEnds; + std::map > pageNumberFields; unsigned textOffsetAccum = 0; std::vector spans; std::vector paras; @@ -1156,6 +1190,13 @@ input->seek(i->offset, librevenge::RVNG_SEEK_SET); tableCellTextEnds[i->id] = parseTableCellDefinitions(input, *i); } + else if (i->name == "TOKN") + { + input->seek(i->offset, librevenge::RVNG_SEEK_SET); + const std::set fields = parsePageNumberFields(input, *i); + if (!fields.empty()) + pageNumberFields[i->id] = fields; + } } if (parsedStrs && parsedSyid && parsedFdpc && parsedFdpp && parsedStsh && parsedFont && textChunkReference != chunkReferences.end()) { @@ -1169,10 +1210,22 @@ std::vector readParas; std::vector readSpans; std::vector text; + const std::map >::const_iterator storyFields = pageNumberFields.find(j); for (unsigned k = 0; k < textLengths[j] && currentTextPara != paras.end() && currentTextSpan != spans.end(); ++k) { - text.push_back(readU8(input)); - text.push_back(readU8(input)); + const unsigned char low = readU8(input); + const unsigned char high = readU8(input); + if (low == '#' && high == 0 && storyFields != pageNumberFields.end() && storyFields->second.count(k)) + { + // PubHarbor: the collector replaces this marker with the page number. + text.push_back(PAGE_NUMBER_FIELD_CHAR & 0xFF); + text.push_back(PAGE_NUMBER_FIELD_CHAR >> 8); + } + else + { + text.push_back(low); + text.push_back(high); + } bytesRead += 2; if (bytesRead >= currentTextSpan->last - textChunkReference->offset) { diff -ruN s1/src/lib/MSPUBParser.h cur/src/lib/MSPUBParser.h --- a/src/lib/MSPUBParser.h 2026-09-25 18:40:32 +++ b/src/lib/MSPUBParser.h 2026-09-25 18:40:32 @@ -130,6 +130,8 @@ &guideData); std::vector parseVertices(const std::vector &vertexData); + std::set parsePageNumberFields(librevenge::RVNGInputStream *input, + const QuillChunkReference &chunk); std::vector parseTableCellDefinitions(librevenge::RVNGInputStream *input, const QuillChunkReference &chunk); std::vector parseSegments( diff -ruN s1/src/lib/MSPUBParser2k.cpp cur/src/lib/MSPUBParser2k.cpp --- a/src/lib/MSPUBParser2k.cpp 2026-09-25 18:40:32 +++ b/src/lib/MSPUBParser2k.cpp 2026-09-25 18:40:32 @@ -20,6 +20,7 @@ #include "Line.h" #include "MSPUBCollector.h" #include "ShapeType.h" +#include "TableInfo.h" #include "libmspub_utils.h" namespace libmspub @@ -56,7 +57,8 @@ m_imageDataChunkIndices(), m_quillColorEntries(), m_chunkChildIndicesById(), - m_chunksBeingRead() + m_chunksBeingRead(), + m_defaultShapesPageSeqNum() { } @@ -369,6 +371,8 @@ input->seek(chunkOffset, librevenge::RVNG_SEEK_SET); unsigned short typeMarker = readU16(input); input->seek(offset, librevenge::RVNG_SEEK_SET); + if (typeMarker == 0x000A && getTableMarker() != 0x000A) + typeMarker = 0xFFFF; // not a table in this version; keep it an unknown chunk switch (typeMarker) { case 0x0014: @@ -401,6 +405,7 @@ case 0x0006: case 0x0007: case 0x0008: + case 0x000A: // PubHarbor: Publisher 2000 table (ignored below for Publisher 97/98) MSPUB_DEBUG_MSG(("Found shape chunk of id 0x%x and parent 0x%x\n", id, parent)); m_contentChunks.push_back(ContentChunkReference(SHAPE, chunkOffset, 0, id, parent)); m_shapeChunkIndices.push_back(unsigned(m_contentChunks.size() - 1)); @@ -437,7 +442,9 @@ return false; } + findDefaultShapesPage(input); + for (unsigned int paletteChunkIndex : m_paletteChunkIndices) { const ContentChunkReference &chunk = m_contentChunks.at(paletteChunkIndex); @@ -540,6 +547,10 @@ { return false; } + if (m_defaultShapesPageSeqNum && get(m_defaultShapesPageSeqNum) == m_contentChunks[i_page].seqNum) + { + return false; + } // if the parent of this is a page and hasn't yet been added, then add it. if (!m_collector->hasPage(chunk.parentSeqNum)) { @@ -707,6 +718,12 @@ { m_collector->setShapeType(seqNum, ELLIPSE); } + else if (int(typeMarker) == getTableMarker()) + { + m_collector->setShapeType(seqNum, RECTANGLE); + isRectangle = true; + parse2kTable(input, seqNum, chunkOffset); + } else if (typeMarker == getTextMarker()) { m_collector->setShapeType(seqNum, RECTANGLE); @@ -720,6 +737,119 @@ unsigned MSPUBParser2k::getTextIdOffset() const { return 0x58; +} + +int MSPUBParser2k::getTableMarker() const +{ + return 0x000A; +} + +// PubHarbor: Publisher 97-2000 store a hidden page that holds one default +// object of each shape type, in ascending type order. libmspub used to rely on +// fixed sequence numbers to skip it, which only match some files; the page +// then rendered as an empty first (or last) page. +void MSPUBParser2k::findDefaultShapesPage(librevenge::RVNGInputStream *input) +{ + if (m_pageChunkIndices.empty()) + return; + const ContentChunkReference &page = m_contentChunks.at(m_pageChunkIndices.front()); + const std::map >::const_iterator it = m_chunkChildIndicesById.find(page.seqNum); + if (it == m_chunkChildIndicesById.end() || it->second.size() < 5) + return; + int previousType = -1; + unsigned run = 0; + for (unsigned childIndex : it->second) + { + const ContentChunkReference &child = m_contentChunks.at(childIndex); + input->seek(child.offset, librevenge::RVNG_SEEK_SET); + const int type = readU16(input); + if (previousType >= 0 && type != previousType + 1) + return; + previousType = type; + ++run; + } + if (run >= 5) + m_defaultShapesPageSeqNum = page.seqNum; +} + +// PubHarbor: Publisher 2000 table chunk (type 0x000A). The column widths and +// row heights follow as (end, size) pairs; a child chunk of type 0x004F lists +// every cell's row and column span. The cell text is the Quill story whose id +// is stored at 0x4E, split into cells by the story's TCD chunk. +bool MSPUBParser2k::parse2kTable(librevenge::RVNGInputStream *input, unsigned seqNum, unsigned chunkOffset) +{ + input->seek(chunkOffset + 0x4E, librevenge::RVNG_SEEK_SET); + const unsigned textId = readU16(input); + const unsigned numColumns = readU16(input); + input->seek(chunkOffset + 0x74, librevenge::RVNG_SEEK_SET); + const unsigned numDimensions = readU16(input); + if (numColumns == 0 || numDimensions <= numColumns || numDimensions > 2048) + return false; + const unsigned numRows = numDimensions - numColumns; + input->seek(chunkOffset + 0x7E, librevenge::RVNG_SEEK_SET); + TableInfo ti(numRows, numColumns); + for (unsigned i = 0; i < numDimensions; ++i) + { + readU32(input); // end position + const unsigned size = readU32(input); + if (i < numColumns) + ti.m_columnWidthsInEmu.push_back(size); + else + ti.m_rowHeightsInEmu.push_back(size); + } + + const std::map >::const_iterator it = m_chunkChildIndicesById.find(seqNum); + if (it != m_chunkChildIndicesById.end()) + { + for (unsigned childIndex : it->second) + { + const ContentChunkReference &child = m_contentChunks.at(childIndex); + input->seek(child.offset, librevenge::RVNG_SEEK_SET); + if (readU16(input) != 0x004F) + continue; + input->seek(child.offset + 0x0A, librevenge::RVNG_SEEK_SET); + const unsigned numCells = readU16(input); + input->seek(child.offset + 0x0E, librevenge::RVNG_SEEK_SET); + const unsigned recordSize = readU16(input); + if (recordSize < 10 || numCells > numRows * numColumns) + break; + std::vector cells; + for (unsigned i = 0; i < numCells; ++i) + { + input->seek(child.offset + 0x10 + i * recordSize + 6, librevenge::RVNG_SEEK_SET); + CellInfo cell; + cell.m_startRow = readU8(input); + cell.m_endRow = readU8(input); + cell.m_startColumn = readU8(input); + cell.m_endColumn = readU8(input); + if (cell.m_startRow > cell.m_endRow || cell.m_endRow >= numRows || + cell.m_startColumn > cell.m_endColumn || cell.m_endColumn >= numColumns) + { + cells.clear(); + break; + } + cells.push_back(cell); + } + ti.m_cells = cells; + break; + } + } + if (ti.m_cells.empty()) + { + for (unsigned row = 0; row < numRows; ++row) + { + for (unsigned column = 0; column < numColumns; ++column) + { + CellInfo cell; + cell.m_startRow = cell.m_endRow = row; + cell.m_startColumn = cell.m_endColumn = column; + ti.m_cells.push_back(cell); + } + } + } + m_collector->setShapeTableInfo(seqNum, ti); + m_collector->addTextShape(textId, seqNum); + return true; } unsigned short MSPUBParser2k::getTextMarker() const diff -ruN s1/src/lib/MSPUBParser2k.h cur/src/lib/MSPUBParser2k.h --- a/src/lib/MSPUBParser2k.h 2026-09-25 18:40:32 +++ b/src/lib/MSPUBParser2k.h 2026-09-25 18:40:32 @@ -27,6 +27,11 @@ std::vector m_quillColorEntries; std::map > m_chunkChildIndicesById; std::deque m_chunksBeingRead; + // PubHarbor: Publisher 97-2000 keep one prototype object of every shape type + // on a hidden page. It is not a page of the publication. + boost::optional m_defaultShapesPageSeqNum; + void findDefaultShapesPage(librevenge::RVNGInputStream *input); + bool parse2kTable(librevenge::RVNGInputStream *input, unsigned seqNum, unsigned chunkOffset); protected: // helper functions @@ -56,6 +61,7 @@ virtual unsigned getShapeFillColorOffset() const; virtual unsigned short getTextMarker() const; virtual unsigned getTextIdOffset() const; + virtual int getTableMarker() const; static Color getColorBy2kIndex(unsigned char index); static Color getColorBy2kHex(unsigned hex); static unsigned translate2kColorReference(unsigned ref2k); diff -ruN s1/src/lib/MSPUBParser97.cpp cur/src/lib/MSPUBParser97.cpp --- a/src/lib/MSPUBParser97.cpp 2026-09-25 18:40:32 +++ b/src/lib/MSPUBParser97.cpp 2026-09-25 18:40:31 @@ -10,6 +10,8 @@ #include "MSPUBParser97.h" #include +#include +#include #include #include #include @@ -37,6 +39,60 @@ return 0x46; } +int MSPUBParser97::getTableMarker() const +{ + return -1; +} + +// PubHarbor: Publisher 97/98 text is stored in a single-byte Windows code page +// and the file records no language for it. ICU's charset detection is +// unreliable on the short strings typical of a publication, so remember the +// code page implied by a Windows 3.x/9x script-specific font family name +// ("Times New Roman Cyr", "Arial CE", ...) in the font table. +void MSPUBParser97::findEncodingHint(librevenge::RVNGInputStream *input) +{ + static const struct + { + const char *suffix; + const char *encoding; + } hints[] = + { + { " Cyr", "windows-1251" }, + { " CE", "windows-1250" }, + { " Greek", "windows-1253" }, + { " Tur", "windows-1254" }, + { " Baltic", "windows-1257" }, + }; + input->seek(0, librevenge::RVNG_SEEK_SET); + std::string contents; + while (!input->isEnd() && contents.size() < m_length) + { + unsigned long numRead = 0; + const unsigned char *buffer = input->read(4096, numRead); + if (!buffer || numRead == 0) + break; + contents.append(reinterpret_cast(buffer), numRead); + } + for (const auto &hint : hints) + { + const std::string needle = std::string(hint.suffix) + '\0'; + size_t position = contents.find(needle); + while (position != std::string::npos) + { + // Require a plausible font name (letters and spaces) before the suffix. + size_t start = position; + while (start > 0 && (std::isalpha(static_cast(contents[start - 1])) || contents[start - 1] == ' ')) + --start; + if (position - start >= 3) + { + m_collector->setFallbackEncoding(hint.encoding); + return; + } + position = contents.find(needle, position + 1); + } + } +} + bool MSPUBParser97::parse() { std::unique_ptr contents(m_input->getSubStreamByName("Contents")); @@ -45,6 +101,7 @@ MSPUB_DEBUG_MSG(("Couldn't get contents stream.\n")); return false; } + findEncodingHint(contents.get()); if (!parseContents(contents.get())) { MSPUB_DEBUG_MSG(("Couldn't parse contents stream.\n")); diff -ruN s1/src/lib/MSPUBParser97.h cur/src/lib/MSPUBParser97.h --- a/src/lib/MSPUBParser97.h 2026-09-25 18:40:32 +++ b/src/lib/MSPUBParser97.h 2026-09-25 18:40:31 @@ -52,6 +52,8 @@ unsigned getShapeFillColorOffset() const override; unsigned short getTextMarker() const override; unsigned getTextIdOffset() const override; + int getTableMarker() const override; + void findEncodingHint(librevenge::RVNGInputStream *input); CharacterStyle readCharacterStyle(librevenge::RVNGInputStream *input, unsigned length); void parseContentsTextIfNecessary(librevenge::RVNGInputStream *input) override;