243 lines
9.2 KiB
Plaintext
243 lines
9.2 KiB
Plaintext
|
#!/usr/bin/python
|
|||
|
|
# -*- coding: utf-8 -*-
|
|||
|
|
""" $Id: htmldiff,v 1.62 2016/10/06 10:46:19 dom Exp $
|
|||
|
|
"""
|
|||
|
|
|
|||
|
|
import atexit
|
|||
|
|
import cgi
|
|||
|
|
import http_auth
|
|||
|
|
import httplib
|
|||
|
|
import os
|
|||
|
|
import re
|
|||
|
|
import surbl
|
|||
|
|
import sys
|
|||
|
|
import tempfile
|
|||
|
|
import tidy
|
|||
|
|
import urlparse
|
|||
|
|
|
|||
|
|
from subprocess import Popen, PIPE
|
|||
|
|
|
|||
|
|
CONTENT_TYPE = "text/html;charset=utf-8"
|
|||
|
|
|
|||
|
|
Page = """
|
|||
|
|
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
|
|||
|
|
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en-US">
|
|||
|
|
<head><title>HTML Diff service</title>
|
|||
|
|
<link rel="stylesheet" href="http://www.w3.org/StyleSheets/base" />
|
|||
|
|
</head>
|
|||
|
|
<body>
|
|||
|
|
|
|||
|
|
<p><a href="http://www.w3.org/"><img src="http://www.w3.org/Icons/w3c_home" alt="W3C"/></a> <a href="http://www.w3.org/2003/Editors">W3C Editors homepage</a></p>
|
|||
|
|
|
|||
|
|
<h1>Create Diff between HTML pages</h1>
|
|||
|
|
"""
|
|||
|
|
Page2 = """
|
|||
|
|
<form method="GET">
|
|||
|
|
<p>Address of reference document: <input name="doc1" type="url" value="%s" style="width:100%%"/></p>
|
|||
|
|
<p>Address of new document: <input name="doc2" value="%s" style="width:100%%"/></p>
|
|||
|
|
<p><input type="submit" value="get Diff"/></p>
|
|||
|
|
</form>
|
|||
|
|
|
|||
|
|
<p><strong>Tip</strong>: if the document uses the W3C convention on linking to its previous version, you can specify only the address of the new document — the previous link will be automatically detected.</p>
|
|||
|
|
<h2>Diff markings</h2>
|
|||
|
|
<p>This service relies on <a href="https://www.gnu.org/software/diffutils/">GNU diff</a>. The found differences are roughly marked as follow:
|
|||
|
|
<ul>
|
|||
|
|
<li>deleted text is shown in pink with down-arrows (as styled for a <del> element)</li>
|
|||
|
|
<li>where there is replacement, it’s shown in green with bi-directional arrows,</li>
|
|||
|
|
<li>where there is newly inserted text, it’s yellow with up arrows (<ins> element)</li>
|
|||
|
|
</ul>
|
|||
|
|
<address>
|
|||
|
|
script $Revision: 1.62 $ of $Date: 2016/10/06 10:46:19 $<br />
|
|||
|
|
by <a href="http://www.w3.org/People/Dom/">Dominique Hazaël-Massieux</a><br />based on <a href="https://dev.w3.org/cvsweb/2009/htmldiff/htmldiff.pl">Shane McCarron’ Perl script</a> wrapped in a <a href="http://dev.w3.org/cvsweb/2009/htmldiff/">Python CGI</a>
|
|||
|
|
</address>
|
|||
|
|
</body>
|
|||
|
|
</html>
|
|||
|
|
"""
|
|||
|
|
|
|||
|
|
def checkInputUrl(url):
|
|||
|
|
checker = surbl.SurblChecker('/usr/local/share/surbl/two-level-tlds','/afs/w3.org/pub/WWW/Systems/Server/debian/generic/usr/local/etc/surbl.whitelist')
|
|||
|
|
|
|||
|
|
if url[:5] == 'file:' or len(urlparse.urlparse(url)[0])<2:
|
|||
|
|
print "Status: 403"
|
|||
|
|
print "Content-Type: text/plain"
|
|||
|
|
print
|
|||
|
|
print "sorry, I decline to handle file: addresses"
|
|||
|
|
sys.exit()
|
|||
|
|
elif checker.isMarkedAsSpam(url):
|
|||
|
|
print "Status: 403"
|
|||
|
|
print "Content-Type: text/plain; charset=utf-8"
|
|||
|
|
print
|
|||
|
|
print "sorry, this URL matches a record known in SURBL. See http://www.surbl.org/"
|
|||
|
|
sys.exit()
|
|||
|
|
|
|||
|
|
def copyHeader(copy_func, source, key, header_name=None):
|
|||
|
|
value = source.get(key)
|
|||
|
|
if not value:
|
|||
|
|
return False
|
|||
|
|
elif header_name is None:
|
|||
|
|
header_name = key
|
|||
|
|
copy_func(header_name, value)
|
|||
|
|
return True
|
|||
|
|
|
|||
|
|
def setupRequest(source_headers):
|
|||
|
|
opener = http_auth.ProxyAuthURLopener()
|
|||
|
|
copyHeader(opener.addheader, source_headers, 'If-Modified-Since')
|
|||
|
|
copyHeader(opener.addheader, os.environ, 'REMOTE_ADDR', 'X_Forward_IP_Addr')
|
|||
|
|
return opener
|
|||
|
|
|
|||
|
|
def tidyFile(file):
|
|||
|
|
# option for tidy
|
|||
|
|
options = dict(tidy_mark=0,show_warnings=0,quiet=1,char_encoding='utf8')
|
|||
|
|
html5 = re.search(r"<!doctype\s+html\s*>", file.read(4096),
|
|||
|
|
re.IGNORECASE)
|
|||
|
|
file.seek(0)
|
|||
|
|
html5_options = {"add_xml_space": "no",
|
|||
|
|
"output_xhtml": "no",
|
|||
|
|
"tidy_mark": "no",
|
|||
|
|
"new_blocklevel_tags": 'article,aside,canvas,dialog,details,figcaption,figure,footer,header,hgroup,menu,nav,section,main,summary,math,semantics,mrow,mfenced,mtable,mtr,mtd,mi,mn,msub,mo,mfrac,munderover,mtext,svg,g,image,rect,text,desc,line,path,polygon,ellipse,tspan,defs,feoffset,fecolormatrix,filter,fegaussianblur,feblend,marker,circle',
|
|||
|
|
"new_inline_tags": 'video,audio,canvas,ruby,rt,rp,time,meter,progress,track,source,emu-val,emu-nt,emu-t,mark',
|
|||
|
|
"break_before_br": "no",
|
|||
|
|
"vertical_space": "no",
|
|||
|
|
"enclose_text": "no",
|
|||
|
|
"numeric_entities": "yes",
|
|||
|
|
"wrap": "1000",
|
|||
|
|
"wrap_attributes": "no",
|
|||
|
|
"drop_empty_paras": "no"
|
|||
|
|
}
|
|||
|
|
if html5:
|
|||
|
|
options.update(html5_options)
|
|||
|
|
newtidy = tidy.parseString(file.read(), **options)
|
|||
|
|
if len(newtidy.errors) > 0:
|
|||
|
|
if not html5:
|
|||
|
|
file.seek(0)
|
|||
|
|
options.update(html5_options)
|
|||
|
|
newtidy = tidy.parseString(file.read(), **options)
|
|||
|
|
file.close()
|
|||
|
|
file = tempfile.NamedTemporaryFile(
|
|||
|
|
mode='w+', prefix='htmldiff-', suffix='.html')
|
|||
|
|
atexit.register(file.close)
|
|||
|
|
file.write(str(newtidy))
|
|||
|
|
file.flush()
|
|||
|
|
file.seek(0)
|
|||
|
|
return (file, newtidy.errors)
|
|||
|
|
|
|||
|
|
def matchPredecessorRel(rel):
|
|||
|
|
return rel and "predecessor-version" in rel.lower().split(" ")
|
|||
|
|
|
|||
|
|
def mirrorURL(url, opener):
|
|||
|
|
try:
|
|||
|
|
filename, headers = opener.retrieve(url)
|
|||
|
|
except IOError, error:
|
|||
|
|
opener.error = "I/O error: %s %s" % (error.errno, error.strerror)
|
|||
|
|
except httplib.InvalidURL:
|
|||
|
|
opener.error = "Invalid URL submitted"
|
|||
|
|
except AttributeError: # ProxyAuthURLopener returned None.
|
|||
|
|
pass # There's already an error set.
|
|||
|
|
else:
|
|||
|
|
atexit.register(os.unlink, filename)
|
|||
|
|
file = open(filename)
|
|||
|
|
if headers.has_key("content-encoding") and headers["content-encoding"] == "gzip":
|
|||
|
|
import gzip
|
|||
|
|
from StringIO import StringIO
|
|||
|
|
data = StringIO(file.read())
|
|||
|
|
file.close()
|
|||
|
|
file = gzip.GzipFile(fileobj=data)
|
|||
|
|
file,errors = tidyFile(file)
|
|||
|
|
if len(errors) == 0:
|
|||
|
|
return (file, headers)
|
|||
|
|
else:
|
|||
|
|
opener.error = "Tidy errors: %s" % (str(errors))
|
|||
|
|
return (None, {})
|
|||
|
|
|
|||
|
|
def showPage(url1='', url2='', error_html='', **headers):
|
|||
|
|
for name, value in headers.items():
|
|||
|
|
print "%s: %s" % (name.replace('_', '-'), value)
|
|||
|
|
print
|
|||
|
|
print Page
|
|||
|
|
print error_html
|
|||
|
|
print Page2 % (url1, url2)
|
|||
|
|
sys.exit()
|
|||
|
|
|
|||
|
|
def serveRequest():
|
|||
|
|
fields = cgi.FieldStorage()
|
|||
|
|
|
|||
|
|
if (not fields.has_key('doc2')):
|
|||
|
|
showPage(Content_Type=CONTENT_TYPE)
|
|||
|
|
# if doc1 is not specified, we load doc2 to check if it has a previous version link
|
|||
|
|
doc2 = fields['doc2'].value
|
|||
|
|
checkInputUrl(doc2)
|
|||
|
|
url_opener2 = setupRequest(fields.headers)
|
|||
|
|
newdoc, newheaders = mirrorURL(doc2, url_opener2)
|
|||
|
|
if fields.has_key('doc1'):
|
|||
|
|
doc1 = fields['doc1'].value
|
|||
|
|
elif newdoc is not None:
|
|||
|
|
from BeautifulSoup import BeautifulSoup
|
|||
|
|
|
|||
|
|
soup = BeautifulSoup(newdoc.read())
|
|||
|
|
newdoc.seek(0)
|
|||
|
|
try:
|
|||
|
|
doc1 = soup.find(text=re.compile("Previous Version",re.IGNORECASE)).findNext(name="a", attrs={"href":True})["href"]
|
|||
|
|
except:
|
|||
|
|
try:
|
|||
|
|
doc1 = soup.find(name=["a", "link"], attrs={"href":True, rel:matchPredecessorRel})["href"]
|
|||
|
|
except:
|
|||
|
|
doc1 = None
|
|||
|
|
else:
|
|||
|
|
doc1 = None
|
|||
|
|
if (not doc1):
|
|||
|
|
showPage(Content_Type=CONTENT_TYPE)
|
|||
|
|
|
|||
|
|
checkInputUrl(doc1)
|
|||
|
|
esc1 = cgi.escape(doc1, True)
|
|||
|
|
esc2 = cgi.escape(doc2, True)
|
|||
|
|
urlcomponents1 = urlparse.urlparse(doc1)
|
|||
|
|
urlcomponents2 = urlparse.urlparse(doc2)
|
|||
|
|
# if same domain, we can use the same urlopener
|
|||
|
|
# otherwise, we create a separate one
|
|||
|
|
if urlcomponents2[1] == urlcomponents1[1]:
|
|||
|
|
url_opener = url_opener2
|
|||
|
|
else:
|
|||
|
|
url_opener = setupRequest(fields.headers)
|
|||
|
|
|
|||
|
|
refdoc, refheaders = mirrorURL(doc1, url_opener)
|
|||
|
|
if not (refdoc and newdoc):
|
|||
|
|
http_error = ""
|
|||
|
|
url = ""
|
|||
|
|
if not refdoc:
|
|||
|
|
http_error = url_opener.error
|
|||
|
|
url = esc1
|
|||
|
|
else:
|
|||
|
|
http_error = url_opener2.error
|
|||
|
|
url = esc2
|
|||
|
|
if re.match("^[1234][0-9][0-9] ", http_error):
|
|||
|
|
print "Status: %s" %(http_error)
|
|||
|
|
error="<p style='color:#FF0000'>An error (%s) occured trying to get <a href='%s'>%s</a>.</p>" % (cgi.escape(http_error), url, url)
|
|||
|
|
showPage(esc1, esc2, error, Content_Type=CONTENT_TYPE)
|
|||
|
|
|
|||
|
|
print "Content-Type: text/html"
|
|||
|
|
if newheaders.has_key('Content-Type'):
|
|||
|
|
contentType = cgi.parse_header(newheaders["Content-Type"])
|
|||
|
|
if contentType[1].has_key('charset'):
|
|||
|
|
charset = contentType[1]['charset'].lower()
|
|||
|
|
#if charset == "iso-8859-1":
|
|||
|
|
# options["char_encoding"]='latin1'
|
|||
|
|
|
|||
|
|
for proxy_header in ('Last-Modified', 'Expires'):
|
|||
|
|
if copyHeader(lambda header, value: sys.stdout.write("%s: %s" %(header, value)), newheaders, proxy_header):
|
|||
|
|
print
|
|||
|
|
print
|
|||
|
|
p = Popen(["/usr/local/bin/htmldiff", refdoc.name, newdoc.name],
|
|||
|
|
stdin=PIPE, stdout=PIPE, stderr=PIPE)
|
|||
|
|
sys.stdout.flush()
|
|||
|
|
sys.stderr.flush()
|
|||
|
|
(out, err) = p.communicate()
|
|||
|
|
p.stdin.close()
|
|||
|
|
if err:
|
|||
|
|
error = "<p style='color:#FF0000'>An error occured when running <code>htmldiff</code> on the documents:</p><pre>%s</pre>" % (cgi.escape(err),)
|
|||
|
|
showPage(esc1, esc2, error)
|
|||
|
|
else:
|
|||
|
|
print out
|
|||
|
|
if __name__ == '__main__':
|
|||
|
|
if os.environ.has_key('SCRIPT_NAME'):
|
|||
|
|
serveRequest()
|