如何在python代码中使用sqoop命令进行增量imp

2024-04-25 07:19:48 发布

您现在位置:Python中文网/ 问答频道 /正文

我想从用户位置_历史进行增量导入,在增量导入后希望保存更新的用户位置_中的最后一个id,以便将来自动执行。在

#!/usr/bin/python
import subprocess
import time
import subprocess
import MySQLdb
import datetime
import sys
import pytz
import os
from subprocess import call

def get_mysql_cursor():
        conn_1 = MySQLdb.connect(user='db', passwd='bazookadb', host='10.216.204.20', db='bazooka')
        conn_2 = MySQLdb.connect(user='db', passwd='bazookadb', host='10.216.204.7', db='bazooka')
        #print conn_1,conn_2
        return conn_1.cursor(),conn_2.cursor()

def get_records():
        cur_1,cur_2 = get_mysql_cursor()
        cur_1.execute("select updated from user_location_updated")
        cur_2.execute("select max(moving_date) from user_location_history")
        return cur_1.fetchone(),cur_2.fetchone()

def update_records(update_date):
        cur_1,cur_2 = get_mysql_cursor()
        print update_date
        query = "update user_location_updated set updated = '"+str(update_date)+"' where id='1' "
        print query
        result = cur_1.execute(query)
        print result



result = get_records()
update_result = update_records(result[1][0])
print result[0][0]
print result[1][0]

sqoopcom = "sqoop import --connect jdbc:mysql://10.216.204.7:3306/bazooka --username db --password bazookadb --fields-terminated-by , --escaped-by \\ --enclosed-by '\"' --table user_location_history  -m 1  --hive-delims-replacement ' ' --as-textfile --incremental append --check-column moving_date --last-value 2016-08-04 19:00:36 --target-dir hdfs://example:9000/user/bigdata/sqoopip --verbose"

#os.system(sqoopcom)
exec (sqoopcom)

----but this code is giving error

Tags: importdbgetdatemysqlupdatelocationresult