. if (next.tagName == "BR") { - var nextElem = this._nextElement(next); + var nextElem = this._nextElement(next.nextSibling); if (nextElem && nextElem.tagName == "BR") break; } + if (!this._isPhrasingContent(next)) break; + // Otherwise, make this node a child of the new
. var sibling = next.nextSibling; p.appendChild(next); next = sibling; } + + while (p.lastChild && this._isWhitespace(p.lastChild)) p.removeChild(p.lastChild); + + if (p.parentNode.tagName === "P") this._setNodeTag(p.parentNode, "DIV"); } }); }, @@ -523,6 +565,7 @@ Readability.prototype = { this._clean(articleContent, "h1"); this._clean(articleContent, "footer"); this._clean(articleContent, "link"); + this._clean(articleContent, "aside"); // Clean out elements have "share" in their id/class combinations from final top candidates, // which means we don't remove the top candidates even they have "share". @@ -579,6 +622,19 @@ Readability.prototype = { if (next && next.tagName == "P") br.parentNode.removeChild(br); }); + + // Remove single-cell tables + this._forEachNode(this._getAllNodesWithTag(articleContent, ["table"]), function(table) { + var tbody = this._hasSingleTagInsideElement(table, "TBODY") ? table.firstElementChild : table; + if (this._hasSingleTagInsideElement(tbody, "TR")) { + var row = tbody.firstElementChild; + if (this._hasSingleTagInsideElement(row, "TD")) { + var cell = row.firstElementChild; + cell = this._setNodeTag(cell, this._everyNode(cell.childNodes, this._isPhrasingContent) ? "P" : "DIV"); + table.parentNode.replaceChild(cell, table); + } + } + }); }, /** @@ -658,37 +714,6 @@ Readability.prototype = { return node && node.nextElementSibling; }, - /** - * Like _getNextNode, but for DOM implementations with no - * firstElementChild/nextElementSibling functionality... - */ - _getNextNodeNoElementProperties: function(node, ignoreSelfAndKids) { - function nextSiblingEl(n) { - do { - n = n.nextSibling; - } while (n && n.nodeType !== n.ELEMENT_NODE); - return n; - } - // First check for kids if those aren't being ignored - if (!ignoreSelfAndKids && node.children[0]) { - return node.children[0]; - } - // Then for siblings... - var next = nextSiblingEl(node); - if (next) { - return next; - } - // And finally, move up the parent chain *and* find a sibling - // (because this is depth-first traversal, we will have already - // seen the parent nodes themselves). - do { - node = node.parentNode; - if (node) - next = nextSiblingEl(node); - } while (node && !next); - return node && next; - }, - _checkByline: function(node, matchString) { if (this._articleByline) { return false; @@ -751,6 +776,12 @@ Readability.prototype = { while (node) { var matchString = node.className + " " + node.id; + if (!this._isProbablyVisible(node)) { + this.log("Removing hidden node - " + matchString); + node = this._removeAndGetNext(node); + continue; + } + // Check to see if this node is a byline, and remove it if it is. if (this._checkByline(node, matchString)) { node = this._removeAndGetNext(node); @@ -784,11 +815,31 @@ Readability.prototype = { // Turn all divs that don't have children block level elements into p's if (node.tagName === "DIV") { + // Put phrasing content into paragraphs. + var p = null; + var childNode = node.firstChild; + while (childNode) { + var nextSibling = childNode.nextSibling; + if (this._isPhrasingContent(childNode)) { + if (p !== null) { + p.appendChild(childNode); + } else if (!this._isWhitespace(childNode)) { + p = doc.createElement('p'); + node.replaceChild(p, childNode); + p.appendChild(childNode); + } + } else if (p !== null) { + while (p.lastChild && this._isWhitespace(p.lastChild)) p.removeChild(p.lastChild); + p = null; + } + childNode = nextSibling; + } + // Sites like http://mobile.slate.com encloses each paragraph with a DIV // element. DIVs with only a P element inside and no text content can be // safely converted into plain P elements to avoid confusing the scoring // algorithm with DIVs with are, in practice, paragraphs. - if (this._hasSinglePInsideElement(node)) { + if (this._hasSingleTagInsideElement(node, "P") && this._getLinkDensity(node) < 0.25) { var newNode = node.children[0]; node.parentNode.replaceChild(newNode, node); node = newNode; @@ -796,17 +847,6 @@ Readability.prototype = { } else if (!this._hasChildBlockElement(node)) { node = this._setNodeTag(node, "P"); elementsToScore.push(node); - } else { - // EXPERIMENTAL - this._forEachNode(node.childNodes, function(childNode) { - if (childNode.nodeType === Node.TEXT_NODE && childNode.textContent.trim().length > 0) { - var p = doc.createElement('p'); - p.textContent = childNode.textContent; - p.style.display = 'inline'; - p.className = 'readability-styled'; - node.replaceChild(p, childNode); - } - }); } } node = this._getNextNode(node); @@ -846,7 +886,7 @@ Readability.prototype = { // Initialize and score ancestors. this._forEachNode(ancestors, function(ancestor, level) { - if (!ancestor.tagName) + if (!ancestor.tagName || !ancestor.parentNode || typeof(ancestor.parentNode.tagName) === 'undefined') return; if (typeof(ancestor.readability) === 'undefined') { @@ -1085,7 +1125,7 @@ Readability.prototype = { // finding the content, and the sieve approach gives us a higher likelihood of // finding the -right- content. var textLength = this._getInnerText(articleContent, true).length; - if (textLength < this._wordThreshold) { + if (textLength < this._charThreshold) { parseSuccessful = false; page.innerHTML = pageCacheHtml; @@ -1233,27 +1273,28 @@ Readability.prototype = { }, /** - * Check if this node has only whitespace and a single P element + * Check if this node has only whitespace and a single element with given tag * Returns false if the DIV node contains non-empty text nodes - * or if it contains no P or more than 1 element. + * or if it contains no element with given tag or more than 1 element. * * @param Element + * @param string tag of child element **/ - _hasSinglePInsideElement: function(element) { - // There should be exactly 1 element child which is a P: - if (element.children.length != 1 || element.children[0].tagName !== "P") { + _hasSingleTagInsideElement: function(element, tag) { + // There should be exactly 1 element child with given tag + if (element.children.length != 1 || element.children[0].tagName !== tag) { return false; } // And there should be no text nodes with real content return !this._someNode(element.childNodes, function(node) { - return node.nodeType === Node.TEXT_NODE && + return node.nodeType === this.TEXT_NODE && this.REGEXPS.hasContent.test(node.textContent); }); }, _isElementWithoutContent: function(node) { - return node.nodeType === Node.ELEMENT_NODE && + return node.nodeType === this.ELEMENT_NODE && node.textContent.trim().length == 0 && (node.children.length == 0 || node.children.length == node.getElementsByTagName("br").length + node.getElementsByTagName("hr").length); @@ -1271,6 +1312,21 @@ Readability.prototype = { }); }, + /*** + * Determine if a node qualifies as phrasing content. + * https://developer.mozilla.org/en-US/docs/Web/Guide/HTML/Content_categories#Phrasing_content + **/ + _isPhrasingContent: function(node) { + return node.nodeType === this.TEXT_NODE || this.PHRASING_ELEMS.indexOf(node.tagName) !== -1 || + ((node.tagName === "A" || node.tagName === "DEL" || node.tagName === "INS") && + this._everyNode(node.childNodes, this._isPhrasingContent)); + }, + + _isWhitespace: function(node) { + return (node.nodeType === this.TEXT_NODE && node.textContent.trim().length === 0) || + (node.nodeType === this.ELEMENT_NODE && node.tagName === "BR"); + }, + /** * Get the inner text of a node - cross browser compatibly. * This also strips out any excess whitespace to be found. @@ -1312,16 +1368,14 @@ Readability.prototype = { if (!e || e.tagName.toLowerCase() === 'svg') return; - if (e.className !== 'readability-styled') { - // Remove `style` and deprecated presentational attributes - for (var i = 0; i < this.PRESENTATIONAL_ATTRIBUTES.length; i++) { - e.removeAttribute(this.PRESENTATIONAL_ATTRIBUTES[i]); - } + // Remove `style` and deprecated presentational attributes + for (var i = 0; i < this.PRESENTATIONAL_ATTRIBUTES.length; i++) { + e.removeAttribute(this.PRESENTATIONAL_ATTRIBUTES[i]); + } - if (this.DEPRECATED_SIZE_ATTRIBUTE_ELEMS.indexOf(e.tagName) !== -1) { - e.removeAttribute('width'); - e.removeAttribute('height'); - } + if (this.DEPRECATED_SIZE_ATTRIBUTE_ELEMS.indexOf(e.tagName) !== -1) { + e.removeAttribute('width'); + e.removeAttribute('height'); } var cur = e.firstElementChild; @@ -1639,6 +1693,10 @@ Readability.prototype = { this._flags = this._flags & ~flag; }, + _isProbablyVisible: function(node) { + return node.style.display != "none" && !node.hasAttribute("hidden"); + }, + /** * Decides whether or not the document is reader-able without parsing the whole thing. * @@ -1663,9 +1721,9 @@ Readability.prototype = { nodes = [].concat.apply(Array.from(set), nodes); } - // FIXME we should have a fallback for helperIsVisible, but this is - // problematic because of jsdom's elem.style handling - see - // https://github.com/mozilla/readability/pull/186 for context. + if (!helperIsVisible) { + helperIsVisible = this._isProbablyVisible; + } var score = 0; // This is a little cheeky, we use the accumulator 'score' to decide what to return from @@ -1719,9 +1777,6 @@ Readability.prototype = { } } - if (typeof this._doc.documentElement.firstElementChild === "undefined") { - this._getNextNode = this._getNextNodeNoElementProperties; - } // Remove script tags from the document. this._removeScripts(this._doc); @@ -1750,7 +1805,6 @@ Readability.prototype = { var textContent = articleContent.textContent; return { - uri: this._uri, title: this._articleTitle, byline: metadata.byline || this._articleByline, dir: this._articleDir, -- cgit v1.2.3