Python：在一定数量的行之后将标准输出打印到多个文件

Question

我有一个数据库表有7000万条记录，另一个表有900万条记录。我试图让我的程序在循环达到1000000条记录后将连接的表打印到多个文件中，并继续这样做直到整个查询转储完成。前提是我的代码：

from netaddr import *
import sys, csv, sqlite3, logging, time, os, errno

# Functions
s = time.strftime('%Y%m%d%H%M%S')

# Create file from standard output for database import
class Logger(object):
    def __init__(self):
        self.terminal = sys.stdout
        self.log = open((s) + "_" + sys.argv[1], "a")

    def write(self, message):
        self.terminal.write(message)
        self.log.write(message)

def ExportVNEDeltas():
    sys.stdout = Logger() # Start screen capture to log file

    con = sqlite3.connect(sys.argv[1])
    cur = con.cursor()

    try:
        cur.execute('SELECT tbl70m.IP, tbl9m.ip, tbl0m.Cgroup, \
            TOTAL(case when tbl9m.vne = "internal" then 1 end), \
            TOTAL(case when tbl9m.vne = "external" then 1 end), \
            TOTAL(case when tbl9m.vne = "enterprise" then 1 end), \
            TOTAL(case when tbl9m.vne = "zone" then 1 end) \
            FROM tbl70m LEFT OUTER JOIN tbl9m ON tbl70m.IP=tbl9m.ip \
            GROUP BY tbl70m.IP \
            ORDER BY tbl70m.Cgroup, tbl70m.IP;')

        data = cur.fetchall()

        for row in data:
            print '"'+str(row[0])+'","'+str(row[1])+'","'+str(row[2])+'","'+str(row[3])+'","'+str(row[4])+'","'+str(row[5])+'","'+str(row[6])+'"'

    except (KeyboardInterrupt, SystemExit):
        raise

    con.close()
    sys.stdout = sys.__stdout__ # stops capturing data from database export.

# Primary function to execute
def main():
    ExportVNEDeltas()

if __name__=='__main__':
    main()

我似乎不知道如何在达到1000000条记录并创建新文件时停止打印标准输出。我需要对1000000进行硬停的原因是这样我们可以在microsoftexcel中查看这些数据。在

Answer 1

您可以将此规则添加到Logger：

class Logger(object):
    def __init__(self, maxlines=1000000):
        self.maxlines = maxlines
        self.lines = 0
        self.terminal = sys.stdout
        self.log = open((s) + "_" + sys.argv[1], "a")

    def write(self, message):
        self.terminal.write(message)
        self.log.write(message)
        self.lines += 1
        if self.lines == self.maxlines:
            self.log.close()
            self.log = open(...)
            self.lines = 0