Как разобрать XML, содержащий HTML
Я пытаюсь разобраться в формате Adium XML в Python. Я хочу создать чистую базу данных чатов, но очищаю все форматирование и гиперссылки.
Я использую xmltodict прямо сейчас, чтобы создавать списки / словари для итерации по нему. Но я сталкиваюсь с серьезными проблемами каждый раз, когда сталкиваюсь с гиперссылкой или форматированием текста. Я думаю, потому что я пытаюсь грубой силой через XML. Он размещает дополнительные теги глубже в списке / словаре.
По сути, я чувствую, что подхожу к этому неправильно.
Вот два фрагмента XML, с которыми я работаю.
XML 1
<?xml version="1.0" encoding="UTF-8" ?>
<chat xmlns="http://purl.org/net/ulf/ns/0.4-02" account="someusername" service="AIM">
<message sender="someusername" time="2008-07-27T18:02:34-0700"><div><span style="font-family: Arial; font-size: 10pt;">time is not of the essence</span></div></message>
<message sender="someusername" time="2008-07-27T18:02:43-0700"><div><span style="font-family: Arial; font-size: 10pt;">it <span style="font-style: italic;">is</span></span><span style="font-family: Helvetica; font-size: 12pt;"> </span><span style="font-family: Arial; font-size: 10pt;">the essence</span></div></message>
<message sender="anotherusername" time="2008-07-27T18:03:49-0700"><div><span style="color: #000000; font-family: Helvetica; font-size: 12pt;">yo</span></div></message>
<message sender="anotherusername" time="2008-07-27T18:03:51-0700"><div><span style="color: #000000; font-family: Helvetica; font-size: 12pt;">whats up?</span></div></message></chat>
XML 2
<?xml version="1.0" encoding="UTF-8" ?>
<chat xmlns="http://purl.org/net/ulf/ns/0.4-02" account="someusername" service="AIM">
<message sender="someusername" time="2009-09-26T05:54:23-0700"><div><a href="http://www.youtube.com/watch?v=LqbJx4TFFEE&feature=related" style="color: #000000; font-family: Helvetica; font-size: 12pt;">http://www.youtube.com/watch?v=LqbJx4TFFEE&feature=related</a></div></message>
<message sender="someusername" time="2009-09-27T16:12:29-0700"><div><span style="color: #000000; font-family: Helvetica; font-size: 12pt;">2nd take, with the bonus stuff I think</span></div></message>
<message sender="someusername" time="2009-09-27T17:18:52-0700"><div></div></message></chat>
И это код, с которым я работал (извините, там есть какая-то чепуха):
import xmltodict
import os
def get_list_of_all_files_in_sub(dirName):
# create a list of file and sub directories
# names in the given directory
listOfFile = os.listdir(dirName)
allFiles = list()
all_files_with_extension = list()
# Iterate over all the entries
for entry in listOfFile:
# Create full path
fullPath = os.path.join(dirName, entry)
# If entry is a directory then get the list of files in this directory
if os.path.isdir(fullPath):
allFiles = allFiles + get_list_of_all_files_in_sub(fullPath)
else:
allFiles.append(fullPath)
return allFiles
def get_files_with_extension(path, file_extension=""):
# gets a list of all files with a certain extension in a folder and all subfolders
files = get_list_of_all_files_in_sub(path)
all_files_with_extension = []
for file in files:
if file.split(".")[-1] == file_extension:
all_files_with_extension.append(file)
return all_files_with_extension
allmessages = []
files = get_files_with_extension("/Users/Desktop/chats", "chatlog")
for file in files:
print (file)
with open(file) as fd:
doc = xmltodict.parse(fd.read())
messages = doc['chat']['message']
# this is gross, but apparently if "messages" only has one entry, it doesn't return a list. So
# to remedy this, im craming it into a list and back into itself to work with the rest of the code.
if type(messages) is not list:
print ("NOT A LIST")
messages_container = messages
messages = [messages_container]
for message in messages:
# Check to see if the SPAN exists inside DIV, which basically is checking to see if there's a real message in it.
if 'span' in message["div"]:
# checks if there's a sender, if there's no sender, it just doesn't include it in the output.
if message["@sender"] != "":
time = (message["@time"])
print (time)
username = (message["@sender"])
print (username)
# SET THE MESSAGE
# If there are multiple messages within one message, then it comes in as a list.
# But so far its just been things like warnings and offline notifications.
# This seems to happen with AIM messages.
if type(message["div"]['span']) is list:
print (message["div"]['span'])
for submessage in message["div"]['span']:
for subsubmessage in submessage:
print ("---------------1----------------")
print (subsubmessage)
print ("---------------2----------------")
if type(subsubmessage) is list:
print (subsubmessage["#text"])
if "Offline IM sent" not in subsubmessage["#text"]:
text_message = (subsubmessage["#text"])
print (text_message)
else:
text_message = (message["div"]['span']["#text"])
print (text_message)
if len(allmessages) > 0:
if (username == allmessages[-1]["sender"]):
if (allmessages[-1]["message"].endswith('.')):
text_message = allmessages[-1]["message"] + " " + text_message
else:
text_message = allmessages[-1]["message"] + ". " + text_message
del allmessages[-1]
newmessage = { 'time' : time,
'sender' : username,
'message' : text_message
}
allmessages.append (newmessage)
#print ("{} {}: {}".format(time, username, message))
for message in x:
print ("{} {}: {}".format(message['time'], message['sender'], message['message']))
Я заметил, что способ, которым xmltodict обрабатывает теги html, превращается в это при выводе:
OrderedDict([('span', OrderedDict([('@style', 'font-family: Arial; font-size: 10pt;'), ('#text', 'time is not of the essence')]))])
OrderedDict([('span', [OrderedDict([('@style', 'font-family: Arial; font-size: 10pt;'), ('span', OrderedDict([('@style', 'font-style: italic;'), ('#text', 'is')])), ('#text', 'it')]), OrderedDict([('@style', 'font-family: Helvetica; font-size: 12pt;')]), OrderedDict([('@style', 'font-family: Arial; font-size: 10pt;'), ('#text', 'the essence')])])])
Как видите, #text с форматированием выдернут и отделен. Любые другие способы или мысли о том, как сделать это, может работать лучше?
0 ответов
Вы можете использовать BeautifulSoup и использовать xml как тип парсера:
from bs4 import BeautifulSoup
import datetime
from pprint import pprint
xml_1 = '''<?xml version="1.0" encoding="UTF-8" ?>
<chat xmlns="http://purl.org/net/ulf/ns/0.4-02" account="someusername" service="AIM">
<message sender="someusername" time="2008-07-27T18:02:34-0700"><div><span style="font-family: Arial; font-size: 10pt;">time is not of the essence</span></div></message>
<message sender="someusername" time="2008-07-27T18:02:43-0700"><div><span style="font-family: Arial; font-size: 10pt;">it <span style="font-style: italic;">is</span></span><span style="font-family: Helvetica; font-size: 12pt;"> </span><span style="font-family: Arial; font-size: 10pt;">the essence</span></div></message>
<message sender="anotherusername" time="2008-07-27T18:03:49-0700"><div><span style="color: #000000; font-family: Helvetica; font-size: 12pt;">yo</span></div></message>
<message sender="anotherusername" time="2008-07-27T18:03:51-0700"><div><span style="color: #000000; font-family: Helvetica; font-size: 12pt;">whats up?</span></div></message></chat>'''
xml_2 = '''<?xml version="1.0" encoding="UTF-8" ?>
<chat xmlns="http://purl.org/net/ulf/ns/0.4-02" account="someusername" service="AIM">
<message sender="someusername" time="2009-09-26T05:54:23-0700"><div><a href="http://www.youtube.com/watch?v=LqbJx4TFFEE&feature=related" style="color: #000000; font-family: Helvetica; font-size: 12pt;">http://www.youtube.com/watch?v=LqbJx4TFFEE&feature=related</a></div></message>
<message sender="someusername" time="2009-09-27T16:12:29-0700"><div><span style="color: #000000; font-family: Helvetica; font-size: 12pt;">2nd take, with the bonus stuff I think</span></div></message>
<message sender="someusername" time="2009-09-27T17:18:52-0700"><div></div></message></chat>'''
def parse_xml(xml_string):
soup = BeautifulSoup(xml_string, 'xml')
data = []
for message in soup.select('message[sender][time]'):
account = message.find_parent('chat')['account']
sender = message['sender']
d = datetime.datetime.strptime(message['time'], "%Y-%m-%dT%H:%M:%S%z") # ISO 8601 time format
text = message.text.strip()
data.append((account, sender, d, text))
return data
pprint(parse_xml(xml_1))
pprint(parse_xml(xml_2))
Печать:
[('someusername',
'someusername',
datetime.datetime(2008, 7, 27, 18, 2, 34, tzinfo=datetime.timezone(datetime.timedelta(-1, 61200))),
'time is not of the essence'),
('someusername',
'someusername',
datetime.datetime(2008, 7, 27, 18, 2, 43, tzinfo=datetime.timezone(datetime.timedelta(-1, 61200))),
'it is the essence'),
('someusername',
'anotherusername',
datetime.datetime(2008, 7, 27, 18, 3, 49, tzinfo=datetime.timezone(datetime.timedelta(-1, 61200))),
'yo'),
('someusername',
'anotherusername',
datetime.datetime(2008, 7, 27, 18, 3, 51, tzinfo=datetime.timezone(datetime.timedelta(-1, 61200))),
'whats up?')]
[('someusername',
'someusername',
datetime.datetime(2009, 9, 26, 5, 54, 23, tzinfo=datetime.timezone(datetime.timedelta(-1, 61200))),
'http://www.youtube.com/watch?v=LqbJx4TFFEE&feature=related'),
('someusername',
'someusername',
datetime.datetime(2009, 9, 27, 16, 12, 29, tzinfo=datetime.timezone(datetime.timedelta(-1, 61200))),
'2nd take, with the bonus stuff I think'),
('someusername',
'someusername',
datetime.datetime(2009, 9, 27, 17, 18, 52, tzinfo=datetime.timezone(datetime.timedelta(-1, 61200))),
'')]