From 7506721e0b2701239cc3b91a2a356a8cda735348 Mon Sep 17 00:00:00 2001 From: zy7y <7631909+zy7y@user.noreply.gitee.com> Date: Tue, 19 Jan 2021 18:55:59 +0800 Subject: [PATCH] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E6=B5=8B=E8=AF=95=E5=89=8D?= =?UTF-8?q?=E6=95=B0=E6=8D=AE=E5=BA=93=E5=A4=87=E4=BB=BD/=E6=B5=8B?= =?UTF-8?q?=E8=AF=95=E5=90=8E=E6=95=B0=E6=8D=AE=E6=81=A2=E5=A4=8D(?= =?UTF-8?q?=E6=94=AF=E6=8C=81linux=E6=9C=8D=E5=8A=A1=E5=99=A8=E9=83=A8?= =?UTF-8?q?=E7=BD=B2mysql=EF=BC=8C=20linux=E4=B8=8Bdocker=E9=83=A8?= =?UTF-8?q?=E7=BD=B2=E7=9A=84mysql=E6=9C=8D=E5=8A=A1)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 8 +- api_server/api.py | 46 ++++++- .../apiAutoTest_2021-01-19T18_14_52.sql | 52 ++++++++ config/config.yaml | 18 ++- data/case_data.xlsx | Bin 31744 -> 35328 bytes test/conftest.py | 16 ++- test/test_api.py | 16 ++- tools/data_clearing.py | 117 ++++++++++++++++++ 8 files changed, 259 insertions(+), 14 deletions(-) create mode 100644 backup_sqls/apiAutoTest_2021-01-19T18_14_52.sql create mode 100644 tools/data_clearing.py diff --git a/README.md b/README.md index 12d2309..426a89c 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,7 @@ | yagmail | 0.11.224 | 测试完成后发送邮件 | | requests| 2.24.0 | 发送请求 | |pymysql|0.10.1|连接mysql| +|paramiko|2.7.2|ssh连接linux服务器，用于备份/删除数据库文件 #### 目录结构 >apiAutoTest > @@ -63,7 +64,7 @@ >> >db.py : 封装连接mysql方法 >> >read_file.py： 读取配置文件、读取excel用例文件 >> > ->> >~~read_data.py：~~ +>> >data_clearing.py: 数据清洗方法封装，ssh2服务器连接，数据库备份/恢复 2021/01/19日更新 >> > >> > >> >send_email.py ： 发送邮件 @@ -134,6 +135,11 @@ https://www.bilibili.com/video/BV1EE411B7SU?p=10 2020/12/08 优化断言信息，增加数据库（支持mysql）查询操作， 使用`@pytest.fixture(scope="session")`来托管数据库对象，用例新增sql栏 2020/12/16 使用conftest.py 初始化用例， 增加失败重跑机制, 增加运行文件run，优化test_api.py冗余代码 + +2021/01/19 添加数据清洗功能(测试开始前进行数据库备份-分别在服务器和本地进行，测试结束后将备份用以恢复数据-将尝试从服务器和本地恢复到服务器数据库中，docker部署的mysql服务已本地调试通过，直接linux部署的mysql并未测试) +> 详细内容见代码注释`tools/data_clearing.py` +> 如不需要使用该功能请做如下处理,如也不使用数据库对象，只需参考 https://gitee.com/zy7y/apiAutoTest/issues/I2BAQL 修改即可 +![](https://gitee.com/zy7y/blog_images/raw/master/img/20210119184856.png) #### 博客园首发 https://www.cnblogs.com/zy7y/p/13426816.html diff --git a/api_server/api.py b/api_server/api.py index 8dcac25..b261097 100644 --- a/api_server/api.py +++ b/api_server/api.py @@ -9,13 +9,24 @@ @desc: 上传文件接口服务,用于调试上传文件接口处理方法，源码来自 FastAPI官网 https://fastapi.tiangolo.com/zh/tutorial/request-files/ """ - +import random from typing import List from fastapi import FastAPI, File, UploadFile +from tools.db import DB + +from faker import Faker + +fake = Faker('zh_CN') + app = FastAPI() +# 连接数据库 +db = DB() +# 创建游标 +cursor = db.connection.cursor() + @app.post("/upload_file/", name='上传单文件接口') async def create_upload_file(file_excel: UploadFile = File(...)): @@ -40,7 +51,40 @@ async def create_upload_files(files: List[UploadFile] = File(...)): return {"filenames": [file.filename for file in files], "meta": {"msg": "ok"}} +@app.post("/users", summary="新增用户") +async def add_user(): + sql = f"insert into user values ({random.randint(10,1000)},'{fake.name()}', '{fake.ean8()}')" + try: + # 执行sql语句 + cursor.execute(sql) + # 提交到数据库执行 + db.connection.commit() + return {"msg": "成功"} + except Exception as e: + # 如果发生错误则回滚 + db.connection.rollback() + print(e) + + + +@app.delete("/users", summary="删除用户") +async def delete_user(id: int): + sql = f"DELETE FROM user WHERE id = {id}" + try: + # 执行sql语句 + cursor.execute(sql) + # 提交到数据库执行 + db.connection.commit() + return {"msg": "成功"} + except Exception as e: + # 如果发生错误则回滚 + db.connection.rollback() + print(e) + + + if __name__ == '__main__': # 启动项目后 访问 http://127.0.0.1:8888/docs 可查看接口文档 import uvicorn + uvicorn.run('api:app', reload=True, port=8888) diff --git a/backup_sqls/apiAutoTest_2021-01-19T18_14_52.sql b/backup_sqls/apiAutoTest_2021-01-19T18_14_52.sql new file mode 100644 index 0000000..548f2c4 --- /dev/null +++ b/backup_sqls/apiAutoTest_2021-01-19T18_14_52.sql @@ -0,0 +1,52 @@ +-- MySQL dump 10.13 Distrib 8.0.22, for Linux (x86_64) +-- +-- Host: 127.0.0.1 Database: apiAutoTest +-- ------------------------------------------------------ +-- Server version 8.0.22 + +/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */; +/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */; +/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */; +/*!50503 SET NAMES utf8mb4 */; +/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */; +/*!40103 SET TIME_ZONE='+00:00' */; +/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */; +/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */; +/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */; +/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */; + +-- +-- Table structure for table `user` +-- + +DROP TABLE IF EXISTS `user`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!50503 SET character_set_client = utf8mb4 */; +CREATE TABLE `user` ( + `id` int NOT NULL AUTO_INCREMENT, + `username` varchar(255) DEFAULT NULL, + `password` varchar(255) DEFAULT NULL, + PRIMARY KEY (`id`) +) ENGINE=InnoDB AUTO_INCREMENT=993 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Dumping data for table `user` +-- + +LOCK TABLES `user` WRITE; +/*!40000 ALTER TABLE `user` DISABLE KEYS */; +INSERT INTO `user` VALUES (3,'所属','1231233'),(604,'薛淑珍','71255132'),(633,'方杰','11881865'),(881,'傅晨','45363849'),(992,'莫英','72334041'); +/*!40000 ALTER TABLE `user` ENABLE KEYS */; +UNLOCK TABLES; +/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */; + +/*!40101 SET SQL_MODE=@OLD_SQL_MODE */; +/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */; +/*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */; +/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */; +/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */; +/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */; +/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */; + +-- Dump completed on 2021-01-19 10:19:19 diff --git a/config/config.yaml b/config/config.yaml index 4cbb85e..70bf912 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -37,17 +37,23 @@ email: database: host: localhost port: 3306 - user: test + user: root # 不用''会被解析成int类型数据 password: '123456' db_name: test charset: utf8mb4 -# 响应存储/sql结果存储 -cache: - host: localhost - port: 6379 - db: 0 + # 数据库所在的服务器配置 + ssh_server: + port: 22 + username: root + password: '123456' + # 私有密钥文件路径 + private_key_file: + # 如果使用的docker容器部署mysql服务，需要传入mysql的容器id/name + mysql_container: mysql8 + # 数据库备份文件导出的本地路径, 需要保证存在该文件夹 + sql_data_file: backup_sqls/ diff --git a/data/case_data.xlsx b/data/case_data.xlsx index 405d8f32b29e02575c2036d2f823fa9782c7a810..d93f7c9ac0fe2b4e7db9a95b6d9373563dfa3666 100644 GIT binary patch delta 7029 zcmeHMX>3&26+ZXPo8@_yH#1(w1{;RW;MxXb7Kg=n%wi09!A`^0+5|7)6dMpVPSOSs zNm|-KLI@WnP1GomMNI=~+DsZnX@OLgT2*SLs%nbVDiKOkHIdq+N>iz7L)`D&w|K^w z{^*ZLl{}+&zjyArXSwH|d+r_k*gwUwv)bv^qTvViKNn)H^lDcZ-$Fzy@T|nM3eRdh z^?26c`E+b-jGsoluXE@}I{tn}A2e>i@K9;X2fq_3c|^ZoyZvtYFMU+JAn*3>k2ov~ z@%jx8R**;sF5ETO_|}mX0M3KOJ+jEXr7*H2b`y&c`J&RpIwSI} zN%gF-_3&D$)sgQO6brAsUXTzz)%s<1;Ue^O6><-^0dD1kg;nzV1*IZS)?;i?4yl&k zL#ijtZB-b)2`VAFD;C2FCCL{~-V-RGdjep%pALHem+O~ieqGoZ7Mo@JoJz4pwJoY` zRc)JU+f}<2ZAoVR+}{_AzsQ;OiK?=G*5wlmj+|`&_+(`7H~M=AzK%qgH)!?WN(0o7 z?=aPp++1H{d_rZ(#e%3?{qg_NM85(_+X470+NJHq_hxe6~L>ujSs&h6Sr6645 zpaw|v(jFzdn`&q=ErpNDR%D{eY>kWMwdR7#WjE?s4igi`u0Z{& z%?xk4s)a0S*F(fCqpN1J3}T06qi!4)D9c3&4v&c%M12 z)9|E&6h&77x)Q)LU=mmj)bZV*oY#>f<4dFGL_}9k%(k7-T_<$g2}FnI&qby|5$(H9imD}>1q$!&3QqYEyX6g1}N+wmfRCa86QcZGoZ^g!D zs!)vskh?Ve!7!rLwnJNF`I9%gauoP);6) zR0=tpDx^|~FaPXa2v2hwJJ2GzbeNcmLkXNzO)kaL+UQc*0S+SpO?OX+VcvB2bQmT~ zB$uU=%aMtsb11fu-t3auCL%dAj3^Ve?npM_?9>QmXwnv!XMl2M7|Aiu_0Oahd!&i@ zZlujl4Kpd*yqRO(%rS4O89L^1;AELMa}4qrlUi+qQOlr9S@!H27R@mj%`q6wF&I?_ z+gvqHfUjZ)??qy##1lBMAEa;6*YF(5`i@c$je=gI#pzYv=EydLij$Q;=de+>;?7nE zpFKP+)IPl8E$}#kQ)0X<;uzw5@Jn)x3-M*qE|2ajujVR5m<3nkfI2}-p6ARi3WXZN$~51E7+DkJW$~gsGBCI*pfq?9 zwJ^Sn5que0BO_~MWQ~j(EHlRBCj)I-<$hc%bdlNmrFR4pvK_-NtCL8l%{P2QYV!z8 zWDDYD<+4CvTBt+G6)6)fptHZ06rYW)xSv{5d^Wbwt+tc~cOa3TAa)tY9!IS;r{7W? zf}7KzBxu1!k0)4!US%>zbJ|kMhu+Oi`fti*lf3+GSRTtH7vL7U z%{IwDa2Z_cN!S*b zO>&89ryaJ*h%&j;Hp$6%qe+(H%F>Q9Y{kY@QoC#^PSpwP&%t3a-a&2j1zRi%v90t) zTZ}XNMq`{Av71hSx$#N4qD598TrNoV9sHS0bS2%pcg3Buv1{8MKhXLk z3a761Q5XGn>Q6iZzXR|V|L+(5gM85!hJ1G4oC>~H@}Erpzpnb_@^gpIy?nSx^Pkds zJI;UHktrJ%n%p*Wi(EJykf%nXvV5dQE*uHqe+{$r68Xl+Mc=VM=i{GanZtLU(*FZt CE6_{; delta 5077 zcmZ`-du)@}6+id$XZ!Obu^pUu(>R1u@+xKw0+c{r#DOLxkd84X8nTp#5W1Brtp;lQ zL7S!zR!F%>tQBbMYwJcuA>O8Hs{m~h+y2?K3ANfZY4G1((!7mH)wC}AoqO#sc5K`7 z_v4&%&+nXj@44rG;lw%P#HV6zv)FL8<<^Nk)Sm56#BHE0flCSOxiL>Sr z-<3U=c6a~fW3f>_X+Cc>AC~_ypA~P(WA3Taj7g*Xr~5;G~A?_T)_UtGRpRmlNgjr`c!A#dQ{zwkxn zv~`b+`Ksg<{N}8vJO}%P?{jhkY(amY8KWv8PkUo>pH-85*x%=_4pB4xdm&8pDTn6I zjo57@6qU))YCA-&1+5`!DyZj)D7GIcDk8H&u~p{j+mt>;Ed_Zh^mv*Pnw#S1yP=<{t$u-EavRb<}Hzn#x_7=~FzhEdBYwd8~!j z*E{-&2|d1DjgRuOYCYxEa_1>;m=ycLH|<9|4X6zeWwoUk^>W_WO{d7g_m` zm6sj{jss_buL4ga{2YkC3A_M&6ZjVJUEq7bOBnM3#{LjkC45vTymSP368M~GOr9Tk z&D`asY7AM0A#vb(;3nW!U>C3l*b7Vm-vfRh_!Hnqz~2J@1pG751m(-1z9_uOV|x#Y z?LI1nsSKt@;0oYs;27`-@G;<5foFhcfo}lc27VX#J>Vta`#@x$Y~JT{*P+cGr&3EdEmEzZvx*2z5{$0cp3O3;D^8;1Ahtp zJMbUCyFjx7(+gsHrN9Vq0edn+H|3rEEq=b>?n41pACN;&?2#9zF3XzHX;G3qGg>0- z&s0qj7LY$$5|FrJwPaAgs$(s=VB<>Gk_pLlBsXm*?o-<4fyCWOOJ3T6WHXWlPOk9J zY!%)Ik^77u>W9RI&Jw4QAnp-b;?(#6X~}@Z)6cJiat`OShQKD!AVk&MKl`M%J$?9c{QWHGdX73n9wll4eWgxk87I1zqZ z(~@Pw$PGh_oCv=~Xo-`-U9Kcv@&wLlM&e}n-HS)MkZM9*>RQT9#)V`O#4iHcmc1%2 zY>&OTEH3Owg7~FG7#@&jR>g%7J5kP>f_FQCHjt6B6ESlVF`ZRmJ`>*om4t~Jaki!( z>g**kRT9fVubnbUm^n#sktJ|&brNoD;h_9ryf(DB32yKX%HNGQjSkU$X*M^eG^A2+ zE4Gzse=gDl-S0?XL%-479!!Z}#xqj(+VZ3Z<=CT5iJf@O)+4yY6X>HEiMiSwiMbwd zq--v);`&Vv7ax!W+LGq7c$Vma8Hu^pI8rv3rMS8dHf`KVYttk6bR~$dD6Rt^xCUsQ zBQe){M*^4p)vG$zn@wKt#20eaAp*qi@;V`i?#e!1m%duY*{Pj5ngX`)H= z%ge{x?L`Zpr!pSm3>`-ladV&W*l>=J^MeY%6Hf-+|+sLHs<;Y2C##f3ri z2@b^`>{bfB{WyJtUZNTN9?$yz4ed8XM(P4ahsjFuc5qKHa0Ei+O3mQkAu>~3D(E4p zuxv1w*#`O5NbG^?g*;*aBhCi3p1m;xY)l}*sUXayfd=M!9~T9lR_Rb-sJKpW%M#qh~d5ckon?*h_S=W|{ zCwO_?uVYZPZ(fw~MA9G5fg8~^@uD0)TrRIX6A?lA(`OpH_^i8-uUjF%6|k(!!G-o6 zP;wB=toI;1V+k!RFkTaritqMhOPgDb_FxQ;X5^$9v9b!;i{)$LRd^l2-~JdW?;I`{ zC32I!Qr1naGUAhb*Cj7b%^7wlZR4C(n>DF3?;BKwE09MQ)>W6Hz^uDbn(SAB_-;tK zU+u-6zeg=4mnC=pZrpm>%g|nSj$;snHE?3LTtb$#l^3(Hm4omS_u{D&5x`|kUApQ03znje{q)FDGDdVA-Q0B z(PeZ<{K<@vgXxfgq9M7wmU7yQAxjFFwUtAb!nzGNoeo*5LJp-vmKF@jUg!ky%DlYb zv?I~sv{#6|md;-nP8e>zv@0Df47WbKVe2Ig7YxSR6e&6Zww4vNif*~;=z8IiqepMb z1NOI*sh7e+K5nm^KT%z)&ad}<~xpPjMf%9(1p8n|_)Qa&;hmfxAF Yk#W 输出结果: {stdout.read().decode()}") + logger.error(f"异常信息: {error}") + return error + + def files_action(self, post: bool, local_path: str = os.getcwd(), remote_path: str = "/root"): + """ + :param post: 动作 为 True 就是上传， False就是下载 + :param local_path: 本地的文件路径， 默认当前脚本所在的工作目录 + :param remote_path: 服务器上的文件路径，默认在/root目录下 + """ + if post: # 上传文件 + self.ftp_client.put(localpath=local_path, remotepath=f"{remote_path}{os.path.split(local_path)[1]}") + logger.info(f"文件上传成功: {local_path} -> {self.host}:{remote_path}{os.path.split(local_path)[1]}") + else: # 下载文件 + file_path = local_path + os.path.split(remote_path)[1] + self.ftp_client.get(remotepath=remote_path, localpath=file_path) + logger.info(f"文件下载成功: {self.host}:{remote_path} -> {file_path}") + + def ssh_close(self): + """关闭连接""" + self.trans.close() + logger.info("已关闭SSH连接...") + + +class DataClearing: + settings = ReadFile.read_config('$.database') + server_settings = settings.get('ssh_server') + server = None + + # 导出的sql文件名称及后缀 + file_name = f"{settings.get('db_name')}_{datetime.now().strftime('%Y-%m-%dT%H_%M_%S')}.sql" + + @classmethod + def server_init(cls, settings=settings, server_settings=server_settings): + cls.server = ServerTools(host=settings.get('host'), port=server_settings.get('port'), + username=server_settings.get('username'), + password=server_settings.get('password'), + private_key_file=server_settings.get('private_key_file')) + # 新建backup_sql文件夹在服务器上，存放导出的sql文件 + cls.server.execute_cmd("mkdir backup_sql") + + @classmethod + def backup_mysql(cls): + """ + 备份数据库, 会分别备份在数据库所在服务器的/root/backup_sql/目录下, 与当前项目文件目录下的 backup_sqls + 每次备份生成一个数据库名_当前年_月_日T_时_分_秒, 支持linux 服务器上安装的mysql服务(本人未调试),以及linux中docker部署的mysql备份 + """ + if cls.server_settings.get('mysql_container') is None: + cmd = f"mysqldump -h127.0.0.1 -u{cls.settings.get('username')} -p{cls.settings.get('password')} {cls.settings.get('db_name')} > {cls.file_name}" + else: + # 将mysql服务的容器中的指定数据库导出， 参考文章 https://www.cnblogs.com/wangsongbai/p/12666368.html + cmd = f"docker exec -i {cls.server_settings.get('mysql_container')} mysqldump -h127.0.0.1 -u{cls.settings.get('user')} -p{cls.settings.get('password')} {cls.settings.get('db_name')} > /root/backup_sql/{cls.file_name}" + cls.server.execute_cmd(cmd) + cls.server.files_action(0, f"{cls.server_settings.get('sql_data_file')}", f"/root/backup_sql/{cls.file_name}") + + @classmethod + def recovery_mysql(cls, sql_file: str = file_name, database: str = settings.get('db_name')): + + """ + 恢复数据库, 从服务器位置(/root/backup_sql/) 或者本地(../backup_sqls)上传, 传入的需要是.sql文件 + :param sql_file: .sql数据库备份文件, 默认就是导出的sql文件名称， 默认文件名称是导出的sql文件 + :param database: 恢复的数据库名称，默认是备份数据库(config.yaml中的db_name) + """ + result = cls.server.execute_cmd(f"ls -l /root/backup_sql/{sql_file}") + if "No such file or directory" in result: + # 本地上传 + cls.server.files_action(1, f"../backup_sqls/{sql_file}", "/root/backup_sql/") + cmd = f"docker exec -i {cls.server_settings.get('mysql_container')} mysql -u{cls.settings.get('user')} -p{cls.settings.get('password')} {database} < /root/backup_sql/{sql_file}" + cls.server.execute_cmd(cmd) + + @classmethod + def close_client(cls): + cls.server.ssh_close()