新增一些py文件，你觉得可以不？其实就算你觉得我不可以，我也要提交上去了。所以没关系的。
This commit is contained in:
parent
e28577bb2f
commit
b28fb4ec3d
|
|
@ -0,0 +1,62 @@
|
|||
# mysql sysbench
|
||||
mysql sysbench是一款压测数据生成工具
|
||||
|
||||
## python3 生成压测数据说明
|
||||
### quick_insert.py
|
||||
|
||||
使用程序方式批量插入数据的脚本，需要引入依赖包 pymysql
|
||||
|
||||
### quick_insert_threads.py
|
||||
|
||||
多线程批量插入数据的脚本，需要引入依赖包 pymysql
|
||||
|
||||
参数配置说明：
|
||||
|
||||
```
|
||||
写入总数为：threads * times * sql_batch
|
||||
offset : 偏移量，默认是0。如果数据库有数据，可以设一个起始点（偏移量）
|
||||
threads : 线程数
|
||||
times : 每个线程执行commit的次数
|
||||
sql_batch : 每次commit多少数据
|
||||
例：要造一亿数据
|
||||
threads = 5
|
||||
times = 2000
|
||||
sql_batch = 10000
|
||||
```
|
||||
|
||||
### quick_sql_insert.py
|
||||
|
||||
生成 insert 批量插入脚本，与上面的方式类似，只不过生成的文件需要手动执行。
|
||||
|
||||
### quick_txt_load.py
|
||||
|
||||
生成逗号分隔的 txt 文件，之后需要使用 `load data infile` 命令导入数据。
|
||||
|
||||
|
||||
## 使用方法
|
||||
### 配置数据源
|
||||
|
||||
```yaml
|
||||
host = "127.0.0.1"
|
||||
port = 3306
|
||||
username = "root"
|
||||
password = "123"
|
||||
charset = "utf-8"
|
||||
db = "test"
|
||||
```
|
||||
### 启动单线程版
|
||||
|
||||
```shell
|
||||
python3 quick_insert.py &
|
||||
```
|
||||
|
||||
### 启动多线程版
|
||||
|
||||
```shell
|
||||
python3 quick_insert_threads.py &
|
||||
```
|
||||
|
||||
# 说明
|
||||
***测试可以直接使用sql/tab.sql建表，也可以改成自己的表结构***
|
||||
|
||||
- [生成mysql亿级压测数据程序](https://blog.csdn.net/evane1890/article/details/112911724)
|
||||
|
|
@ -0,0 +1,142 @@
|
|||
# -*- coding:utf-8 -*-
|
||||
|
||||
import uuid
|
||||
import random
|
||||
import time
|
||||
from datetime import datetime
|
||||
import pymysql as mysql
|
||||
|
||||
host = "127.0.0.1"
|
||||
port = 3306
|
||||
username = "root"
|
||||
password = "123"
|
||||
charset = "utf-8"
|
||||
db = "test"
|
||||
|
||||
# 数据起始点
|
||||
start_id = 1
|
||||
|
||||
|
||||
class QuickInsert(object):
|
||||
def __init__(self):
|
||||
super(QuickInsert, self).__init__()
|
||||
|
||||
@staticmethod
|
||||
def connect():
|
||||
return mysql.connect(host=host, port=port, user=username, passwd=password, db=db)
|
||||
|
||||
def strTimeProp(self, start, end, prop, frmt):
|
||||
stime = time.mktime(time.strptime(start, frmt))
|
||||
etime = time.mktime(time.strptime(end, frmt))
|
||||
ptime = stime + prop * (etime - stime)
|
||||
return int(ptime)
|
||||
|
||||
def randomTimestamp(self, frmt='%Y-%m-%d %H:%M:%S'):
|
||||
start = '2016-06-02 12:12:12'
|
||||
end = '2020-07-27 00:00:00'
|
||||
return self.strTimeProp(start, end, random.random(), frmt)
|
||||
|
||||
def randomDate(self, frmt='%Y-%m-%d'):
|
||||
start = '2016-06-02'
|
||||
end = '2020-07-27'
|
||||
return self.strTimeProp(start, end, random.random(), frmt)
|
||||
|
||||
def createPhone(self):
|
||||
for k in range(10):
|
||||
prelist = ["130", "131", "132", "133", "134", "135", "136", "137", "138", "139",
|
||||
"147", "150", "151", "152", "153", "155", "156", "157", "158", "159",
|
||||
"186", "187", "188", "189"]
|
||||
return random.choice(prelist) + "".join(random.choice("0123456789") for i in range(8))
|
||||
|
||||
def getCardName(self):
|
||||
for k in range(10):
|
||||
prelist = ["圣诞咖啡快乐会员星礼包红杯字母款", "大杯香草风味星冰乐电子券", "星巴克星情月饼礼盒",
|
||||
"大杯冷萃冰咖啡(手机银行)", "2020 年Roastery 专用咖啡乐园券", "大杯摩卡", "实用主义商品福袋",
|
||||
"香草风味拿铁大杯电子券", "星巴克夏日活力早餐套餐电子券", "摩卡可可碎片星冰乐"]
|
||||
return random.choice(prelist)
|
||||
|
||||
def getCardStatus(self):
|
||||
for k in range(10):
|
||||
prelist = ["01-已使用", "04-未激活", "03-未激活", "06-激活", "10-系统作废",
|
||||
"0-已使用", "20-作废"]
|
||||
return random.choice(prelist)
|
||||
|
||||
def getIsReplace(self):
|
||||
for k in range(10):
|
||||
prelist = ["Y", "N"]
|
||||
return random.choice(prelist)
|
||||
|
||||
|
||||
def insert_data(self):
|
||||
# start_id = self.id
|
||||
cursor = self.conn.cursor()
|
||||
global start_id
|
||||
print("开始于：", start_id)
|
||||
|
||||
for x in range(5000):
|
||||
|
||||
# 表，换成自己的。测试用的建表语句见 tab.sql
|
||||
insert_user_sql = """
|
||||
INSERT INTO T_BALANCE_REPORT (`ID`,`ACCOUNT_ID`,`CARD_NO`,`CARD_NAME`,`CARD_STATUS`,`UPDATE_DATE`,`TRADING_DATE`,`BALANCE`,`INIT_AMOUNT`,`INIT_DATE`,`VALID_END_DATE`,`IS_REPLACE`,`OLD_CARD_NO`)
|
||||
VALUES ( %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s );
|
||||
"""
|
||||
|
||||
user_values, order_values = [], []
|
||||
# user_values = []
|
||||
for i in range(10000):
|
||||
|
||||
timestamp = self.randomTimestamp()
|
||||
time_local = time.localtime(timestamp)
|
||||
createTime = time.strftime("%Y-%m-%d %H:%M:%S", time_local)
|
||||
radomDate = time.strftime("%Y-%m-%d", time_local)
|
||||
user_id = str(uuid.uuid4())
|
||||
|
||||
user_values.append((
|
||||
# --------------- 参数 --------
|
||||
# NULL,
|
||||
# 1001358650,
|
||||
# '1001358650',
|
||||
# '压测数据',
|
||||
# '压测激活',
|
||||
# '2021-01-15 00:00:00',
|
||||
# '2020-01-15 00:00:00',
|
||||
# 100.000,
|
||||
# 100.000,
|
||||
# '20210115',
|
||||
# '20200115',
|
||||
# '0',
|
||||
# '1001358650'
|
||||
# --------------- 参数 --------
|
||||
start_id,
|
||||
start_id,
|
||||
start_id,
|
||||
self.getCardName() + str(start_id),
|
||||
self.getCardStatus(),
|
||||
createTime,
|
||||
createTime,
|
||||
random.randint(1, 30000) / random.randint(1, 17),
|
||||
random.randint(100, 10000),
|
||||
radomDate,
|
||||
radomDate,
|
||||
self.getIsReplace(),
|
||||
# self.createPhone()
|
||||
"".join(random.choice("0123456789") for i in range(8))
|
||||
))
|
||||
start_id = start_id + 1
|
||||
|
||||
cursor.executemany(insert_user_sql, user_values)
|
||||
# cursor.executemany(insert_user_sql, order_values) #可以插入多个表
|
||||
self.conn.commit()
|
||||
print("写入：", (x * 10000) + start_id)
|
||||
|
||||
cursor.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
quickInsert = QuickInsert()
|
||||
quickInsert.conn = QuickInsert.connect()
|
||||
startTime = datetime.now()
|
||||
print("开始时间", startTime)
|
||||
quickInsert.insert_data()
|
||||
endTime = datetime.now()
|
||||
print("结束时间", endTime, "共持续", (endTime - startTime).seconds, "秒")
|
||||
|
|
@ -0,0 +1,152 @@
|
|||
# -*- coding:utf-8 -*-
|
||||
|
||||
import uuid
|
||||
import random
|
||||
import time
|
||||
from datetime import datetime
|
||||
import pymysql as mysql
|
||||
from threading import Thread, current_thread
|
||||
|
||||
host = "127.0.0.1"
|
||||
port = 3306
|
||||
username = "svc"
|
||||
password = "svc123"
|
||||
charset = "utf-8"
|
||||
db = "svc"
|
||||
|
||||
class QuickInsert(object):
|
||||
def __init__(self):
|
||||
super(QuickInsert, self).__init__()
|
||||
|
||||
@staticmethod
|
||||
def connect():
|
||||
return mysql.connect(host=host, port=port, user=username, passwd=password, db=db)
|
||||
|
||||
def strTimeProp(self, start, end, prop, frmt):
|
||||
stime = time.mktime(time.strptime(start, frmt))
|
||||
etime = time.mktime(time.strptime(end, frmt))
|
||||
ptime = stime + prop * (etime - stime)
|
||||
return int(ptime)
|
||||
|
||||
def randomTimestamp(self, frmt='%Y-%m-%d %H:%M:%S'):
|
||||
start = '2016-06-02 12:12:12'
|
||||
end = '2020-07-27 00:00:00'
|
||||
return self.strTimeProp(start, end, random.random(), frmt)
|
||||
|
||||
def randomDate(self, frmt='%Y-%m-%d'):
|
||||
start = '2015-06-02'
|
||||
end = '2021-01-21'
|
||||
return self.strTimeProp(start, end, random.random(), frmt)
|
||||
|
||||
def createPhone(self):
|
||||
for k in range(10):
|
||||
prelist = ["130", "131", "132", "133", "134", "135", "136", "137", "138", "139",
|
||||
"147", "150", "151", "152", "153", "155", "156", "157", "158", "159",
|
||||
"186", "187", "188", "189"]
|
||||
return random.choice(prelist) + "".join(random.choice("0123456789") for i in range(8))
|
||||
|
||||
def getCardName(self):
|
||||
for k in range(10):
|
||||
prelist = ["圣诞咖啡快乐会员星礼包红杯字母款", "大杯香草风味星冰乐电子券", "星巴克星情月饼礼盒",
|
||||
"大杯冷萃冰咖啡(手机银行)", "2020 年Roastery 专用咖啡乐园券", "大杯摩卡", "实用主义商品福袋",
|
||||
"香草风味拿铁大杯电子券", "星巴克夏日活力早餐套餐电子券", "摩卡可可碎片星冰乐"]
|
||||
return random.choice(prelist)
|
||||
|
||||
def getCardStatus(self):
|
||||
for k in range(10):
|
||||
prelist = ["01-已使用", "04-未激活", "03-未激活", "06-激活", "10-系统作废",
|
||||
"0-已使用", "20-作废"]
|
||||
return random.choice(prelist)
|
||||
|
||||
def getIsReplace(self):
|
||||
for k in range(10):
|
||||
prelist = ["Y", "N"]
|
||||
return random.choice(prelist)
|
||||
|
||||
def insert_data(self, times, sql_b, s_id):
|
||||
startTime = datetime.now()
|
||||
print(current_thread().getName(), "开始时间", startTime)
|
||||
|
||||
start_id = s_id
|
||||
sql_batch = sql_b
|
||||
print(current_thread().getName() + "开始于：" + str(start_id) + "，times：" + str(times))
|
||||
cursor = self.conn.cursor()
|
||||
|
||||
for x in range(times):
|
||||
insert_user_sql = """
|
||||
INSERT INTO T_BALANCE_REPORT (`ID`,`ACCOUNT_ID`,`CARD_NO`,`CARD_NAME`,`CARD_STATUS`,`UPDATE_DATE`,`TRADING_DATE`,`BALANCE`,`INIT_AMOUNT`,`INIT_DATE`,`VALID_END_DATE`,`IS_REPLACE`,`OLD_CARD_NO`)
|
||||
VALUES ( %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s );
|
||||
"""
|
||||
|
||||
user_values, order_values = [], []
|
||||
# user_values = []
|
||||
for i in range(sql_batch):
|
||||
|
||||
timestamp = self.randomTimestamp()
|
||||
time_local = time.localtime(timestamp)
|
||||
createTime = time.strftime("%Y-%m-%d %H:%M:%S", time_local)
|
||||
radomDate = time.strftime("%Y-%m-%d", time_local)
|
||||
user_id = str(uuid.uuid4())
|
||||
|
||||
user_values.append((
|
||||
# 参数
|
||||
start_id,
|
||||
start_id,
|
||||
start_id,
|
||||
self.getCardName() + str(start_id),
|
||||
self.getCardStatus(),
|
||||
createTime,
|
||||
createTime,
|
||||
random.randint(1, 30000) / random.randint(1, 17),
|
||||
random.randint(100, 10000),
|
||||
radomDate,
|
||||
radomDate,
|
||||
self.getIsReplace(),
|
||||
# self.createPhone()
|
||||
"".join(random.choice("0123456789") for i in range(8))
|
||||
))
|
||||
|
||||
start_id = start_id + 1
|
||||
|
||||
cursor.executemany(insert_user_sql, user_values)
|
||||
# cursor.executemany(insert_user_sql, order_values)
|
||||
self.conn.commit()
|
||||
print(current_thread().getName() + "--写入点：" + str((x * 10000) + start_id))
|
||||
|
||||
cursor.close()
|
||||
endTime = datetime.now()
|
||||
print(current_thread().getName(), "结束时间", endTime, "共持续", (endTime - startTime).seconds, "秒")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
#########################################
|
||||
# 写入总数为：threads * times * sql_batch #
|
||||
# offset : 偏移量，默认是0。如果数据库有数据，可以设一个起始点（偏移量）
|
||||
# threads : 线程数
|
||||
# times : 每个线程执行commit的次数
|
||||
# sql_batch : 每次commit多少数据
|
||||
# 例：要造一亿数据
|
||||
# threads = 5
|
||||
# times = 2000
|
||||
# sql_batch = 10000
|
||||
#########################################
|
||||
|
||||
# 下次运行的起始位置 = (threads * times * sql_batch) + offset
|
||||
offset = 1
|
||||
threads = 3
|
||||
times = 10
|
||||
sql_batch = 10000
|
||||
|
||||
print("----------------本次写入总数：" + str(threads * times * sql_batch))
|
||||
|
||||
for i in range(threads):
|
||||
start_id = (i * times * sql_batch) + offset
|
||||
quickInsert = QuickInsert()
|
||||
quickInsert.conn = QuickInsert.connect()
|
||||
|
||||
t = Thread(target=quickInsert.insert_data, args=(times, sql_batch, start_id))
|
||||
t.start()
|
||||
# t.join() # 同步操作
|
||||
|
||||
|
||||
|
|
@ -0,0 +1,74 @@
|
|||
# -*- coding:utf-8 -*-
|
||||
|
||||
import uuid
|
||||
import random
|
||||
import time
|
||||
from datetime import datetime
|
||||
|
||||
|
||||
class QuickSqlBuilder(object):
|
||||
def __init__(self):
|
||||
super(QuickSqlBuilder, self).__init__()
|
||||
|
||||
def strTimeProp(self, start, end, prop, frmt):
|
||||
stime = time.mktime(time.strptime(start, frmt))
|
||||
etime = time.mktime(time.strptime(end, frmt))
|
||||
ptime = stime + prop * (etime - stime)
|
||||
return int(ptime)
|
||||
|
||||
def randomTimestamp(self, frmt='%Y-%m-%d %H:%M:%S'):
|
||||
start = '2016-06-02 12:12:12'
|
||||
end = '2020-07-27 00:00:00'
|
||||
return self.strTimeProp(start, end, random.random(), frmt)
|
||||
|
||||
def createPhone(self):
|
||||
for k in range(10):
|
||||
prelist = ["130", "131", "132", "133", "134", "135", "136", "137", "138", "139",
|
||||
"147", "150", "151", "152", "153", "155", "156", "157", "158", "159",
|
||||
"186", "187", "188", "189"]
|
||||
return random.choice(prelist) + "".join(random.choice("0123456789") for i in range(8))
|
||||
|
||||
def insert_data(self):
|
||||
with open("./sql/insert_user_500w.sql", "w+") as f, open("./sql/insert_order_500w+.sql", "w+") as f1:
|
||||
for x in range(1000):
|
||||
insert_user_sql = "insert into `user` ( `id`,`user_name`,`age`, `phone`, `province`, `city`, `create_time`,`update_time` ) values "
|
||||
insert_order_sql = "insert into `order` ( `id`, `product_count`, `user_id`, `price`, `create_time`, `update_time`) values "
|
||||
user_values, order_values = [], []
|
||||
for i in range(5000):
|
||||
timestamp = self.randomTimestamp()
|
||||
time_local = time.localtime(timestamp)
|
||||
create_time = time.strftime("%Y-%m-%d %H:%M:%S", time_local)
|
||||
user_id = str(uuid.uuid4())
|
||||
user_values.append(
|
||||
(user_id, "名字" + str(x) + str(i),
|
||||
random.randint(1, 120), self.createPhone(),
|
||||
str(random.randint(1, 26)),
|
||||
str(random.randint(1, 1000)), create_time,
|
||||
create_time))
|
||||
random_order_count = random.randint(0, 3)
|
||||
if random_order_count > 0:
|
||||
for c in range(random_order_count):
|
||||
timestamp = self.randomTimestamp()
|
||||
time_local = time.localtime(timestamp)
|
||||
order_create_time = time.strftime("%Y-%m-%d %H:%M:%S", time_local)
|
||||
order_values.append((str(uuid.uuid4()), random.randint(1, 5), user_id,
|
||||
random.randint(10, 2000), order_create_time, order_create_time))
|
||||
|
||||
user_values_str = str(user_values).strip('[]')
|
||||
insert_user_sql += user_values_str
|
||||
f.write(insert_user_sql)
|
||||
f.write(";\n")
|
||||
|
||||
order_values_str = str(order_values).strip('[]')
|
||||
insert_order_sql += order_values_str
|
||||
f1.write(insert_order_sql)
|
||||
f1.write(";\n")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
quickInsert = QuickSqlBuilder()
|
||||
startTime = datetime.now()
|
||||
print("开始时间", startTime)
|
||||
quickInsert.insert_data()
|
||||
endTime = datetime.now()
|
||||
print("结束时间", endTime, "共持续", (endTime - startTime).seconds, "秒")
|
||||
|
|
@ -0,0 +1,68 @@
|
|||
# -*- coding:utf-8 -*-
|
||||
|
||||
import uuid
|
||||
import random
|
||||
import time
|
||||
from datetime import datetime
|
||||
|
||||
|
||||
class QuickTxtBuilder(object):
|
||||
def __init__(self):
|
||||
super(QuickTxtBuilder, self).__init__()
|
||||
|
||||
def strTimeProp(self, start, end, prop, frmt):
|
||||
stime = time.mktime(time.strptime(start, frmt))
|
||||
etime = time.mktime(time.strptime(end, frmt))
|
||||
ptime = stime + prop * (etime - stime)
|
||||
return int(ptime)
|
||||
|
||||
def randomTimestamp(self, frmt='%Y-%m-%d %H:%M:%S'):
|
||||
start = '2016-06-02 12:12:12'
|
||||
end = '2020-07-27 00:00:00'
|
||||
return self.strTimeProp(start, end, random.random(), frmt)
|
||||
|
||||
def createPhone(self):
|
||||
for k in range(10):
|
||||
prelist = ["130", "131", "132", "133", "134", "135", "136", "137", "138", "139",
|
||||
"147", "150", "151", "152", "153", "155", "156", "157", "158", "159",
|
||||
"186", "187", "188", "189"]
|
||||
return random.choice(prelist) + "".join(random.choice("0123456789") for i in range(8))
|
||||
|
||||
def insert_data(self):
|
||||
with open("./sql/load_user_txt_500w.txt", "w+") as f, open("./sql/load_order_txt_500w+.txt", "w+") as f1:
|
||||
for x in range(1000):
|
||||
user_row_str, order_row_str = '', ''
|
||||
for i in range(5000):
|
||||
timestamp = self.randomTimestamp()
|
||||
time_local = time.localtime(timestamp)
|
||||
create_time = time.strftime("%Y-%m-%d %H:%M:%S", time_local)
|
||||
user_id = str(uuid.uuid4())
|
||||
user_row = "{0},{1},{2},{3},{4},{5},{6},{7}".format(user_id, "名字" + str(x) + str(i),
|
||||
random.randint(1, 120), self.createPhone(),
|
||||
str(random.randint(1, 26)),
|
||||
str(random.randint(1, 1000)), create_time,
|
||||
create_time)
|
||||
|
||||
random_order_count = random.randint(0, 3)
|
||||
if random_order_count > 0:
|
||||
for c in range(random_order_count):
|
||||
timestamp = self.randomTimestamp()
|
||||
time_local = time.localtime(timestamp)
|
||||
order_create_time = time.strftime("%Y-%m-%d %H:%M:%S", time_local)
|
||||
order_row = "{0},{1},{2},{3},{4},{5}".format(str(uuid.uuid4()), user_id, random.randint(1, 5),
|
||||
random.randint(10, 2000), order_create_time, order_create_time)
|
||||
order_row_str += str(order_row) + "\n"
|
||||
|
||||
user_row_str += str(user_row) + "\n"
|
||||
|
||||
f.write(user_row_str)
|
||||
f1.write(order_row_str)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
quickInsert = QuickTxtBuilder()
|
||||
startTime = datetime.now()
|
||||
print("开始时间", startTime)
|
||||
quickInsert.insert_data()
|
||||
endTime = datetime.now()
|
||||
print("结束时间", endTime, "共持续", (endTime - startTime).seconds, "秒")
|
||||
Loading…
Reference in New Issue