數(shù)據(jù)編碼與處理
讀寫csv文件
import csv
from collections import namedtuple
def main1():
with open('stocks.csv') as f:
f_csv = csv.reader(f)
#跳過表頭
headers = next(f_csv)
for row in f_csv:
print (row)
def main2():
with open('stocks.csv') as f:
f_csv = csv.reader(f)
headings = next(f_csv)
Row = namedtuple('Row', headings)
for r in f_csv:
row = Row(*r)
print (row)
def main3():
with open('stocks.csv') as f:
f_csv = csv.DictReader(f)
for r in f_csv:
print (r)
- 在使用命名元組時(shí),需要處理表頭頭非法字符的情況比如'-',使用正則進(jìn)行替換
def main5():
import re
with open('stocks.csv') as f:
f_csv = csv.reader(f)
headers = [re.sub(r'[^a-zA-Z]', '_', h) for h in next(f_csv)]
Row = namedtuple('Row', headers)
for r in f_csv:
row = Row(*r)
print (row)
def write_1():
headers = ['Symbol','Price','Date','Time','Change','Volume']
rows = [
('AA', 39.48, '6/11/2007', '9:36am', -0.18, 181800),
('AIG', 71.38, '6/11/2007', '9:36am', -0.15, 195500),
('AXP', 62.58, '6/11/2007', '9:36am', -0.46, 935000),
]
with open('stocks.csv', 'w') as f:
f_csv = csv.writer(f)
f_csv.writerow(headers)
for row in rows:
f_csv.writerow(row)
def write_2():
headers = ['Symbol', 'Price', 'Date', 'Time', 'Change', 'Volume']
rows = [
{'Symbol':'AA', 'Price':39.48, 'Date':'6/11/2007','Time':'9:36am', 'Change':-0.18, 'Volume':181800},
{'Symbol':'AIG', 'Price': 71.38, 'Date':'6/11/2007','Time':'9:36am', 'Change':-0.15, 'Volume': 195500},
{'Symbol':'AXP', 'Price': 62.58, 'Date':'6/11/2007','Time':'9:36am', 'Change':-0.46, 'Volume': 935000},
]
with open('stocks.csv', 'w') as f:
f_csv = csv.DictWriter(f, headers)
f_csv.writeheader()
for row in rows:
f_csv.writerow(row)
- 改變編碼的讀取規(guī)則,例如以tab鍵分隔的csv
def main4():
with open('stocks.csv') as f:
#tab 分隔
f_csv = csv.reader(f, delimiter='\t')
for row in f_csv:
print (row)
- csv是不會(huì)對(duì)數(shù)據(jù)進(jìn)行額外處理,需要自行處理
def convert_1():
col_types = [str, float, str, str, float, int]
with open('stocks.csv') as f:
f_csv = csv.reader(f)
headers = next(f_csv)
for row in f_csv:
row = tuple(convert(value) for convert, value in zip(col_types, row))
print (row)
def convert_2():
field_types = [
('Price', float),
('Change', float),
('Volume', int)
]
with open('stocks.csv') as f:
for row in csv.DictReader(f):
#每行逐次掃描,轉(zhuǎn)換數(shù)據(jù)更新到字典中
row.update((key, conversion(row[key])) for key, conversion in field_types)
print (row)
讀寫Josn數(shù)據(jù)
- json 解碼會(huì)解碼出字典或者列表,在loads時(shí)傳遞object_pairs_hook或object_hook參數(shù),可以解碼成需要的對(duì)象
- 分別將json字符串解碼成OrderedDict和JSONObject對(duì)象
s= '{"name": "ACME", "shares": 50, "price": 490.1}'
>>> from collections import OrderedDict
>>> json.loads(s, object_pairs_hook=OrderedDict)
OrderedDict([('name', 'ACME'), ('shares', 50), ('price', 490.1)])
>>> class JSONObject:
... def __init__(self, d):
... self.__dict__ = d
>>> data = json.loads(s, object_hook=JSONObject)
>>> data.name
'ACME'
>>>
>>> print (json.dumps({'a':1}, indent=4))
{
"a": 1
}
- 序列化類實(shí)例,通過函數(shù)將類實(shí)例轉(zhuǎn)化為字典
import json
class Point:
def __init__(self, x, y):
self.x = x
self.y = y
def serialize_instance(obj):
d = {'__classname__':type(obj).__name__}
d.update(vars(obj))
return d
if __name__ == '__main__':
p = Point(2, 3)
s = json.dumps(p, default=serialize_instance)
print (s)
解析簡單的xml
- parse()將整個(gè)xml文檔解析為文檔對(duì)象,就可以利用find查詢特定的信息
>>> from urllib.request import urlopen
>>> from xml.etree.ElementTree import parse
>>> u = urlopen('http://planet.python.org/rss20.xml')
>>> doc = parse(u)
>>> doc
<xml.etree.ElementTree.ElementTree object at 0x10dcd42e8>
>>> e = doc.find('channel/title')
>>> e.tag
'title'
>>> e.text
'Planet Python'
將字典轉(zhuǎn)化為xml
- 如果需要保持dict元素的順序,需要使用OrdereDict對(duì)象
from xml.etree.ElementTree import Element, tostring
def dict_to_xml(tag, d):
#創(chuàng)建最外層的節(jié)點(diǎn)
elem = Element(tag)
for key, val in d.items():
child = Element(key)
child.text = str(val)
#最外層節(jié)點(diǎn)上添加內(nèi)容
elem.append(child)
return elem
if __name__ == '__main__':
s = {'name':'GOOG', 'shares':100, 'price':490.1}
e = dict_to_xml('stock', s)
#給原始添加屬性
e.set('_id', '1234')
print (tostring(e))
#輸出
b'<stock _id="1234"><name>GOOG</name><shares>100</shares><price>490.1</price></stock>'
def dict_to_xml_str(tag, d):
parts = ['<{}>'.format(tag)]
for key, val in d.items():
parts.append('<{0}>{1}<0>'.format(key, val))
parts.append('</{}>'.format(tag))
return ''.join(parts)
if __name__ == '__main__':
s = {'name':'GOOG', 'shares':100, 'price':490.1}
e = dict_to_xml_str('stock', s)
print (e)
#輸出
<stock><name>GOOG<0><shares>100<0><price>490.1<0></stock>
編碼和解碼十六進(jìn)制數(shù)
- 字節(jié)字符串和十六進(jìn)制的編碼或解碼
- base64中的16進(jìn)制轉(zhuǎn)換只能操作大寫形式
>>> s = b'hello'
>>> import binascii
>>> h = binascii.b2a_hex(s)
>>> h
b'68656c6c6f'
>>> binascii.a2b_hex(h)
b'hello'
>>> import base64
>>> h = base64.b16encode(s)
>>> h
b'68656C6C6F'
>>> base64.b16decode(h)
b'hello'
#編碼為Unicode
>>> h = h.decode('ascii')
>>> h
'68656C6C6F'
encode(編碼) decode(解碼) base64
>>> import base64
>>> s = b'hello'
>>> a = base64.b64encode(s)
>>> a
b'aGVsbG8='
>>> base64.b64decode(a)
b'hello'
#解碼為unicode
>>> base64.b64decode(a).decode('ascii')
'hello'
讀寫二進(jìn)制數(shù)組數(shù)據(jù)
- 寫入元組到二進(jìn)制文本中
- 使用struct編碼寫入
from struct import Struct
def write_records(records, format, f):
record_struct = Struct(format)
for r in records:
f.write(record_struct.pack(*r))
if __name__ == '__main__':
#write
records = [
(1, 2.3, 4.5),
(6, 7.8, 9.0),
(12, 13.4, 56.7),
]
with open('data.b', 'wb') as f:
#小端存儲(chǔ) int double double
write_records(records, '<idd', f)
- 以增量形式讀取二進(jìn)制數(shù)據(jù)
def read_records(format, f):
record_struct = Struct(format)
#沒有形參的lambda,不斷的生成size大小的迭代器,直到b''為止
chunks = iter(lambda: f.read(record_struct.size), b'')
#生成器可以迭代三次,每次20字節(jié)
return (record_struct.unpack(chunk) for chunk in chunks)
if __name__ == '__main__':
#read
with open('data.b', 'rb') as f:
for r in read_records('<idd', f):
print (r)
def unpack_records(format, data):
'''
設(shè)置好解包步長,一次解包返回迭代器
'''
record_struct = Struct(format)
return (record_struct.unpack_from(data, offset) \
for offset in range(0, len(data), record_struct.size))
if __name__ == '__main__':
#read
with open('data.b', 'rb') as f:
data = f.read()
for rec in unpack_records('<idd', data):
print (rec)
