user3738031 user3738031 - 2 months ago 7
Python Question

How to write list in Apache Thrift?

I need your help.
I would like to test Apache Thrift on a simple task.
I have a few tables in MySQL database and I want to save the stored data into a file.

So, I created a Thrift IDL file as follows:

exception InvalidOperation {
1: i32 what,
2: string why,
}

struct Product{
1: i32 id
2: i16 resource_id,
3: i16 resource_amount,
4: double price,
5: string comment,
6: string comment_text_color,
7: string comment_bgr_color,
8: string image_file_id,
9: i16 active,
10: string title,
11: string description,
12: i16 item_index,
13: i16 is_special,
14: string banner_file_id
}

struct File{
1: string id
2: string path,
3: string checksum,
4: string size,
}

struct StaticData{
1: list<Product> Products = [];
2: list<File> Files = [];
}

service DataLibrary {
void ping(),
}


and wrote some simple python code:
server.py:

# -*- coding: utf-8 -*-

import thriftpy

from thriftpy.rpc import make_server

tutorial_thrift = thriftpy.load("tutorial.thrift",
module_name="tutorial_thrift")

class StaticDataHandler(object):
def __init__(self):
self.log = {}

def ping(self):
print('ping()')

def main():
server = make_server(tutorial_thrift.StaticData, StaticDataHandler(),
'127.0.0.1', 6000)
print("serving...")
server.serve()

if __name__ == '__main__':
main()


client.py:

# -*- coding: utf-8 -*-
import sys
import thriftpy

sys.path.append("gen-py")
from thriftpy.rpc import client_context
from thrift.transport import TTransport
from thrift.protocol import TBinaryProtocol
from thrift import Thrift
from tutorial import ttypes

tutorial_thrift = thriftpy.load("tutorial.thrift",
module_name="tutorial_thrift")


def get_products_from_db():
product_list = []

for i in range(1, 500):
product_list.append({'id': i,
'resource_id': i,
'resource_amount': i,
'price': i,
'comment': 'comment' + str(i),
'comment_text_color': 'yellow',
'comment_bgr_color': 'blue',
'image_file_id': 'file_' + str(i),
'active': 1,
'title': 'title_' + str(i),
'description': 'description' + str(i),
'item_index': i,
'is_special': 1,
'banner_file_id': 'banner_' + str(i)})
return product_list


def main():
with client_context(tutorial_thrift.DataLibrary,
'127.0.0.1', 6000) as client:
client.ping()
print("ping()")
product_list = get_products_from_db()
static_data = tutorial_thrift.StaticData()
static_data.Products.append(product_list)
static_data_save_to_file(static_data)


def static_data_save_to_file(data):
trans = TTransport.TFileObjectTransport(open("static_data", "wb"))
proto = TBinaryProtocol.TBinaryProtocol(trans)
static_data = tutorial_thrift.StaticData()
proto.writeStructBegin(static_data)
proto.writeListBegin(Thrift.TType.LIST, len(data.Products))
print(data.Products)
# proto.writeContainerList(data.Products, Thrift.TType.LIST)
proto.writeFieldEnd()
proto.writeFieldStop()
proto.writeStructEnd()
print('Recording finished')


def product_save_to_file():
trans = TTransport.TFileObjectTransport(open("data", "wb"))
proto = TBinaryProtocol.TBinaryProtocol(trans)
product_list = get_products_from_db()
product = product_list[0]

print(product)
print('Starting record data to file')
proto.writeStructBegin(product)

proto.writeFieldBegin("id", Thrift.TType.I32, 1)
proto.writeI32(product.id)
proto.writeFieldEnd()

proto.writeFieldBegin("resource_id", Thrift.TType.I16, 2)
proto.writeI16(product.resource_id)
proto.writeFieldEnd()

proto.writeFieldBegin("resource_amount", Thrift.TType.I16, 3)
proto.writeI16(product.resource_amount)
proto.writeFieldEnd()

proto.writeFieldBegin("price", Thrift.TType.DOUBLE, 4)
proto.writeDouble(product.price)
proto.writeFieldEnd()

proto.writeFieldBegin("comment", Thrift.TType.STRING, 5)
proto.writeString(product.comment)
proto.writeFieldEnd()

proto.writeFieldBegin("comment_text_color", Thrift.TType.STRING, 6)
proto.writeString(product.comment_text_color)
proto.writeFieldEnd()

proto.writeFieldBegin("comment_bgr_color", Thrift.TType.STRING, 7)
proto.writeString(product.comment_bgr_color)
proto.writeFieldEnd()

proto.writeFieldBegin("image_file_id", Thrift.TType.STRING, 8)
proto.writeString(product.image_file_id)
proto.writeFieldEnd()

proto.writeFieldBegin("active", Thrift.TType.I16, 9)
proto.writeI16(product.active)
proto.writeFieldEnd()

proto.writeFieldBegin("title", Thrift.TType.STRING, 10)
proto.writeString(product.title)
proto.writeFieldEnd()

proto.writeFieldBegin("description", Thrift.TType.STRING, 11)
proto.writeString(product.description)
proto.writeFieldEnd()

proto.writeFieldBegin("item_index", Thrift.TType.I16, 12)
proto.writeI16(product.item_index)
proto.writeFieldEnd()

proto.writeFieldBegin("is_special", Thrift.TType.I16, 13)
proto.writeI16(product.is_special)
proto.writeFieldEnd()

proto.writeFieldBegin("banner_file_id", Thrift.TType.STRING, 14)
proto.writeString(product.banner_file_id)
proto.writeFieldEnd()

proto.writeFieldStop()
proto.writeStructEnd()
print('Recording finished')


def product_read_from_file():
trans = TTransport.TFileObjectTransport(open("data", "rb"))
trans.open()
proto = TBinaryProtocol.TBinaryProtocol(trans)
ro = ttypes.Product()
ro.read(proto)
print('started')
# Display the contents of the Product
print("\nReading Product")
print("-------------------------")
print("resource_id : %d" % ro.resource_id)
print("resource_amount : %s" % ro.resource_amount)
print("Price : %f" % ro.price)
print('end')

if __name__ == '__main__':
main()


I get stuck in function static_data_save_to_file on this line:

# proto.writeContainerList(data.Products, Thrift.TType.LIST)


I can't understand, which attributes I should send to this method, and how can I can count size.

I tried to save just one product to file and read it, and it works, but now I need to insert a few lists (Products, Files) in one stucture (StaticData) and save it in a file.

Answer

I think you are trying to do in a much too complicated way. I have not tested the following, but something like this should do:

def static_data_save_to_file(data):
    trans = TTransport.TFileObjectTransport(open("static_data", "wb"))
    proto = TBinaryProtocol.TBinaryProtocol(trans)
    data.write(proto)
    print('Recording finished')