Skip to content

W
wangxiaolu-link-python-clean-data
提交 ea335516 authored 作者: lidongxu's avatar lidongxu
浏览文件
操作

包一层文件夹

上级 fa6c4c14
隐藏空白字符变更
内嵌 并排
正在显示 包含 363 行增加 和 109 行删除
code/bi_price_xx.sql 0 → 100644
浏览文件 @ ea335516
/*
Navicat MySQL Data Transfer
Source Server : t100_production
Source Server Version : 50744
Source Host : rm-2ze28qp55mrm34g8bbo.mysql.rds.aliyuncs.com:3306
Source Database : market_bi
Target Server Type : MYSQL
Target Server Version : 50744
File Encoding : 65001
Date: 2026-03-12 11:37:31
*/
SET FOREIGN_KEY_CHECKS=0;
-- ----------------------------
-- Table structure for bi_price_xx
-- ----------------------------
DROP TABLE IF EXISTS `bi_price_xx`;
CREATE TABLE `bi_price_xx` (
`id` varchar(255) COLLATE utf8mb4_bin DEFAULT NULL COMMENT '主键',
`bi_product` varchar(255) COLLATE utf8mb4_bin DEFAULT NULL COMMENT '产品系统',
`prd_name` varchar(255) COLLATE utf8mb4_bin DEFAULT NULL COMMENT '口味',
`pro_weight` varchar(255) COLLATE utf8mb4_bin DEFAULT NULL COMMENT '产品克重',
`channel_type` varchar(255) COLLATE utf8mb4_bin DEFAULT NULL COMMENT '渠道',
`creator` varchar(255) COLLATE utf8mb4_bin DEFAULT NULL COMMENT '提交人',
`modifier` varchar(255) COLLATE utf8mb4_bin DEFAULT NULL COMMENT '修改人',
`creator_nickname` varchar(255) COLLATE utf8mb4_bin DEFAULT NULL COMMENT '提交人昵称',
`modifier_nickname` varchar(255) COLLATE utf8mb4_bin DEFAULT NULL COMMENT '修改人昵称',
`create_time` datetime DEFAULT NULL COMMENT '提交时间',
`modify_time` datetime DEFAULT NULL COMMENT '修改时间',
`qbi_system_upload_id` bigint(30) DEFAULT NULL COMMENT '上传批次主键',
`low_price` decimal(30,2) DEFAULT NULL COMMENT '低价',
`normal_price` decimal(30,2) DEFAULT NULL COMMENT '零售价'
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin COMMENT='线下价盘表';
-- ----------------------------
-- Records of bi_price_xx
-- ----------------------------
INSERT INTO `bi_price_xx` VALUES ('7ac70b27-59d8-413c-81e4-d11d3e753ca2', '虎皮凤爪', '全品味', '105g', 'KA', '86f47c35e2d4477d838e1280a949028b', '86f47c35e2d4477d838e1280a949028b', '王璐璐', '王璐璐', '2026-03-12 11:31:37', '2026-03-12 11:31:37', '377410', '14.90', '21.89');
INSERT INTO `bi_price_xx` VALUES ('3ebea3f4-4e40-4088-aacb-d5d7b3194d82', '虎皮凤爪', '全品味', '210g', 'KA', '86f47c35e2d4477d838e1280a949028b', '86f47c35e2d4477d838e1280a949028b', '王璐璐', '王璐璐', '2026-03-12 11:31:37', '2026-03-12 11:31:37', '377410', '26.50', '38.39');
INSERT INTO `bi_price_xx` VALUES ('344df49c-597d-4826-b659-2fd1638edc45', '虎皮凤爪', '全品味', '散称', 'KA', '86f47c35e2d4477d838e1280a949028b', '86f47c35e2d4477d838e1280a949028b', '王璐璐', '王璐璐', '2026-03-12 11:31:37', '2026-03-12 11:31:37', '377410', '45.80', '54.78');
INSERT INTO `bi_price_xx` VALUES ('21f14713-eb58-4090-8df0-aa8a5b98f3e8', '去骨凤爪', '全品味', '72g', 'KA', '86f47c35e2d4477d838e1280a949028b', '86f47c35e2d4477d838e1280a949028b', '王璐璐', '王璐璐', '2026-03-12 11:31:37', '2026-03-12 11:31:37', '377410', '14.90', '21.89');
INSERT INTO `bi_price_xx` VALUES ('7d222a14-8191-4f88-9fec-2fa0e21c27cc', '去骨凤爪', '全品味', '138g', 'KA', '86f47c35e2d4477d838e1280a949028b', '86f47c35e2d4477d838e1280a949028b', '王璐璐', '王璐璐', '2026-03-12 11:31:37', '2026-03-12 11:31:37', '377410', '26.50', '38.39');
INSERT INTO `bi_price_xx` VALUES ('3d2ced09-1e88-46ef-87a3-e2368e5aaff1', '脆笋去骨', '全品味', '散称', 'KA', '86f47c35e2d4477d838e1280a949028b', '86f47c35e2d4477d838e1280a949028b', '王璐璐', '王璐璐', '2026-03-12 11:31:37', '2026-03-12 11:31:37', '377410', '45.80', '50.38');
INSERT INTO `bi_price_xx` VALUES ('bcc8e320-0db0-4535-9fd0-3e79585881c8', '老卤凤爪', '全品味', '95g', 'KA', '86f47c35e2d4477d838e1280a949028b', '86f47c35e2d4477d838e1280a949028b', '王璐璐', '王璐璐', '2026-03-12 11:31:37', '2026-03-12 11:31:37', '377410', '14.90', '21.89');
INSERT INTO `bi_price_xx` VALUES ('f795e572-7b8a-41f9-b2a4-678803019224', '老卤鸭掌', '全品味', '95g', 'KA', '86f47c35e2d4477d838e1280a949028b', '86f47c35e2d4477d838e1280a949028b', '王璐璐', '王璐璐', '2026-03-12 11:31:37', '2026-03-12 11:31:37', '377410', '14.90', '21.89');
INSERT INTO `bi_price_xx` VALUES ('de4bcae9-6ace-43b1-b88f-7c70ca674b64', '鸡肉豆堡', '全品味', '120g', 'KA', '86f47c35e2d4477d838e1280a949028b', '86f47c35e2d4477d838e1280a949028b', '王璐璐', '王璐璐', '2026-03-12 11:31:37', '2026-03-12 11:31:37', '377410', '9.90', '14.19');
INSERT INTO `bi_price_xx` VALUES ('4949ad7f-8f90-4bf9-8848-e862b210e3e0', '虎皮小鸡腿', '全品味', '80g', 'KA', '86f47c35e2d4477d838e1280a949028b', '86f47c35e2d4477d838e1280a949028b', '王璐璐', '王璐璐', '2026-03-12 11:31:37', '2026-03-12 11:31:37', '377410', '12.80', '18.48');
INSERT INTO `bi_price_xx` VALUES ('4286df2e-dcf4-4dd5-a25e-c26d8a3c8829', '虎皮凤爪', '全品味', '105g', 'BC', '86f47c35e2d4477d838e1280a949028b', '86f47c35e2d4477d838e1280a949028b', '王璐璐', '王璐璐', '2026-03-12 11:31:37', '2026-03-12 11:31:37', '377410', '14.90', '21.89');
INSERT INTO `bi_price_xx` VALUES ('e8b863b9-ce10-4a33-b142-6019c62aee93', '虎皮凤爪', '全品味', '210g', 'BC', '86f47c35e2d4477d838e1280a949028b', '86f47c35e2d4477d838e1280a949028b', '王璐璐', '王璐璐', '2026-03-12 11:31:37', '2026-03-12 11:31:37', '377410', '26.50', '38.39');
INSERT INTO `bi_price_xx` VALUES ('07ddcf94-619e-468a-9ad7-67b1835c5898', '虎皮凤爪', '全品味', '68g', 'BC', '86f47c35e2d4477d838e1280a949028b', '86f47c35e2d4477d838e1280a949028b', '王璐璐', '王璐璐', '2026-03-12 11:31:37', '2026-03-12 11:31:37', '377410', '9.80', '14.19');
INSERT INTO `bi_price_xx` VALUES ('0a193d95-e8ec-4e12-abe0-5dc465b1e1bd', '虎皮凤爪', '全品味', '25g', 'BC', '86f47c35e2d4477d838e1280a949028b', '86f47c35e2d4477d838e1280a949028b', '王璐璐', '王璐璐', '2026-03-12 11:31:37', '2026-03-12 11:31:37', '377410', '3.90', '5.39');
INSERT INTO `bi_price_xx` VALUES ('347aa860-eadf-4ae0-aa31-7a64f15322b3', '虎皮凤爪', '全品味', '散称', 'BC', '86f47c35e2d4477d838e1280a949028b', '86f47c35e2d4477d838e1280a949028b', '王璐璐', '王璐璐', '2026-03-12 11:31:37', '2026-03-12 11:31:37', '377410', '45.80', '54.78');
INSERT INTO `bi_price_xx` VALUES ('24e265c7-d526-4b30-a981-2eae56ac2227', '去骨凤爪', '全品味', '72g', 'BC', '86f47c35e2d4477d838e1280a949028b', '86f47c35e2d4477d838e1280a949028b', '王璐璐', '王璐璐', '2026-03-12 11:31:37', '2026-03-12 11:31:37', '377410', '14.90', '21.89');
INSERT INTO `bi_price_xx` VALUES ('71a70334-e06a-43a0-913e-e81d1eafcf94', '老卤凤爪', '全品味', '95g', 'BC', '86f47c35e2d4477d838e1280a949028b', '86f47c35e2d4477d838e1280a949028b', '王璐璐', '王璐璐', '2026-03-12 11:31:37', '2026-03-12 11:31:37', '377410', '14.90', '21.89');
INSERT INTO `bi_price_xx` VALUES ('b9686a81-9771-4c4b-8c4f-faf0b5fdf71f', '老卤鸭掌', '全品味', '95g', 'BC', '86f47c35e2d4477d838e1280a949028b', '86f47c35e2d4477d838e1280a949028b', '王璐璐', '王璐璐', '2026-03-12 11:31:37', '2026-03-12 11:31:37', '377410', '14.90', '21.89');
INSERT INTO `bi_price_xx` VALUES ('350cc768-1407-40ec-b9c4-4a336cd89118', '鸡肉豆堡', '全品味', '120g', 'BC', '86f47c35e2d4477d838e1280a949028b', '86f47c35e2d4477d838e1280a949028b', '王璐璐', '王璐璐', '2026-03-12 11:31:37', '2026-03-12 11:31:37', '377410', '9.90', '14.19');
INSERT INTO `bi_price_xx` VALUES ('5c67fa66-799b-43bc-8263-b84291ec7c44', '鸡肉豆堡', '全品味', '散称', 'BC', '86f47c35e2d4477d838e1280a949028b', '86f47c35e2d4477d838e1280a949028b', '王璐璐', '王璐璐', '2026-03-12 11:31:37', '2026-03-12 11:31:37', '377410', '34.90', '38.39');
INSERT INTO `bi_price_xx` VALUES ('09084915-3a6a-4339-a8b1-f26ca803aec6', '虎皮小鸡腿', '全品味', '80g', 'BC', '86f47c35e2d4477d838e1280a949028b', '86f47c35e2d4477d838e1280a949028b', '王璐璐', '王璐璐', '2026-03-12 11:31:37', '2026-03-12 11:31:37', '377410', '12.80', '18.48');
INSERT INTO `bi_price_xx` VALUES ('54b65a1e-41b2-4d86-9895-64eb03f33ed0', '虎皮凤爪', '全品味', '105g', 'CVS', '86f47c35e2d4477d838e1280a949028b', '86f47c35e2d4477d838e1280a949028b', '王璐璐', '王璐璐', '2026-03-12 11:31:37', '2026-03-12 11:31:37', '377410', '14.90', '21.89');
INSERT INTO `bi_price_xx` VALUES ('44b67fcf-42ac-4b93-83e0-da427b013737', '虎皮凤爪', '全品味', '210g', 'CVS', '86f47c35e2d4477d838e1280a949028b', '86f47c35e2d4477d838e1280a949028b', '王璐璐', '王璐璐', '2026-03-12 11:31:37', '2026-03-12 11:31:37', '377410', '29.50', '38.39');
INSERT INTO `bi_price_xx` VALUES ('d5dfc92a-a075-4277-938b-269f103aa76f', '虎皮凤爪', '全品味', '68g', 'CVS', '86f47c35e2d4477d838e1280a949028b', '86f47c35e2d4477d838e1280a949028b', '王璐璐', '王璐璐', '2026-03-12 11:31:37', '2026-03-12 11:31:37', '377410', '10.90', '14.19');
INSERT INTO `bi_price_xx` VALUES ('ce9500ff-6671-4514-8a40-6a84c79ab1a3', '虎皮凤爪', '全品味', '25g', 'CVS', '86f47c35e2d4477d838e1280a949028b', '86f47c35e2d4477d838e1280a949028b', '王璐璐', '王璐璐', '2026-03-12 11:31:37', '2026-03-12 11:31:37', '377410', '3.90', '5.39');
INSERT INTO `bi_price_xx` VALUES ('0a34f1bc-7e5e-4cf3-a059-da9a29d0bad8', '去骨凤爪', '全品味', '72g', 'CVS', '86f47c35e2d4477d838e1280a949028b', '86f47c35e2d4477d838e1280a949028b', '王璐璐', '王璐璐', '2026-03-12 11:31:37', '2026-03-12 11:31:37', '377410', '14.90', '21.89');
INSERT INTO `bi_price_xx` VALUES ('0bf423ef-5275-43ce-80b4-dac3662b21c2', '去骨凤爪', '全品味', '138g', 'CVS', '86f47c35e2d4477d838e1280a949028b', '86f47c35e2d4477d838e1280a949028b', '王璐璐', '王璐璐', '2026-03-12 11:31:37', '2026-03-12 11:31:37', '377410', '29.50', '38.39');
INSERT INTO `bi_price_xx` VALUES ('0ebf0fe1-25cd-4b1c-bb16-698d714a90f6', '老卤凤爪', '全品味', '95g', 'CVS', '86f47c35e2d4477d838e1280a949028b', '86f47c35e2d4477d838e1280a949028b', '王璐璐', '王璐璐', '2026-03-12 11:31:37', '2026-03-12 11:31:37', '377410', '14.90', '21.89');
INSERT INTO `bi_price_xx` VALUES ('3456b8c9-772f-4c8b-a2a1-a2562273839f', '老卤鸭掌', '全品味', '95g', 'CVS', '86f47c35e2d4477d838e1280a949028b', '86f47c35e2d4477d838e1280a949028b', '王璐璐', '王璐璐', '2026-03-12 11:31:37', '2026-03-12 11:31:37', '377410', '14.90', '21.89');
INSERT INTO `bi_price_xx` VALUES ('55957f41-d285-4dac-b661-b1c6f7faf4c1', '鸡肉豆堡', '全品味', '120g', 'CVS', '86f47c35e2d4477d838e1280a949028b', '86f47c35e2d4477d838e1280a949028b', '王璐璐', '王璐璐', '2026-03-12 11:31:37', '2026-03-12 11:31:37', '377410', '9.90', '14.19');
INSERT INTO `bi_price_xx` VALUES ('44274c3b-697b-401b-9670-b663a235cbf9', '虎皮小鸡腿', '全品味', '80g', 'CVS', '86f47c35e2d4477d838e1280a949028b', '86f47c35e2d4477d838e1280a949028b', '王璐璐', '王璐璐', '2026-03-12 11:31:37', '2026-03-12 11:31:37', '377410', '12.80', '18.48');
INSERT INTO `bi_price_xx` VALUES ('fb961ee5-3d1c-406b-81f0-68f73a24e82a', '虎皮凤爪', '全品味', '68g', '零食', '86f47c35e2d4477d838e1280a949028b', '86f47c35e2d4477d838e1280a949028b', '王璐璐', '王璐璐', '2026-03-12 11:31:37', '2026-03-12 11:31:37', '377410', '8.80', '14.19');
INSERT INTO `bi_price_xx` VALUES ('e400d8e8-19ce-41dd-b02e-391933d76d43', '虎皮凤爪', '全品味', '散称', '零食', '86f47c35e2d4477d838e1280a949028b', '86f47c35e2d4477d838e1280a949028b', '王璐璐', '王璐璐', '2026-03-12 11:31:37', '2026-03-12 11:31:37', '377410', '45.80', '54.78');
INSERT INTO `bi_price_xx` VALUES ('edacdcb9-6450-4965-870c-979f5aa0fdf3', '脆笋去骨', '全品味', '散称', '零食', '86f47c35e2d4477d838e1280a949028b', '86f47c35e2d4477d838e1280a949028b', '王璐璐', '王璐璐', '2026-03-12 11:31:37', '2026-03-12 11:31:37', '377410', '45.80', '50.38');
INSERT INTO `bi_price_xx` VALUES ('d75c4276-56e6-4296-b389-8ea2ff386d48', '鸡肉豆堡', '全品味', '散称', '零食', '86f47c35e2d4477d838e1280a949028b', '86f47c35e2d4477d838e1280a949028b', '王璐璐', '王璐璐', '2026-03-12 11:31:37', '2026-03-12 11:31:37', '377410', '34.90', '38.39');
INSERT INTO `bi_price_xx` VALUES ('b748e150-82ad-447d-89f7-7a5b4efc5b25', '虎皮小鸡腿', '全品味', '散称', '零食', '86f47c35e2d4477d838e1280a949028b', '86f47c35e2d4477d838e1280a949028b', '王璐璐', '王璐璐', '2026-03-12 11:31:37', '2026-03-12 11:31:37', '377410', '35.80', '39.38');
INSERT INTO `bi_price_xx` VALUES ('a8854d47-e1a3-4673-afb5-9b7d0fa8d183', '虎皮凤爪', '全品味', '105g', '批发', '86f47c35e2d4477d838e1280a949028b', '86f47c35e2d4477d838e1280a949028b', '王璐璐', '王璐璐', '2026-03-12 11:31:37', '2026-03-12 11:31:37', '377410', '10.40', '9.90');
INSERT INTO `bi_price_xx` VALUES ('b93e7fef-8ecf-4434-9441-da61b839069b', '虎皮凤爪', '全品味', '210g', '批发', '86f47c35e2d4477d838e1280a949028b', '86f47c35e2d4477d838e1280a949028b', '王璐璐', '王璐璐', '2026-03-12 11:31:37', '2026-03-12 11:31:37', '377410', '19.11', '18.20');
INSERT INTO `bi_price_xx` VALUES ('cc8f1359-2afc-464c-8f0f-62e985e99fb7', '虎皮凤爪', '全品味', '68g', '批发', '86f47c35e2d4477d838e1280a949028b', '86f47c35e2d4477d838e1280a949028b', '王璐璐', '王璐璐', '2026-03-12 11:31:37', '2026-03-12 11:31:37', '377410', '6.83', '6.50');
INSERT INTO `bi_price_xx` VALUES ('5711f296-0451-48c6-bcf7-efdb68d5867b', '虎皮凤爪', '全品味', '25g', '批发', '86f47c35e2d4477d838e1280a949028b', '86f47c35e2d4477d838e1280a949028b', '王璐璐', '王璐璐', '2026-03-12 11:31:37', '2026-03-12 11:31:37', '377410', '2.63', '2.50');
INSERT INTO `bi_price_xx` VALUES ('aab9599c-2a33-437d-8ef8-890ed1bec685', '虎皮凤爪', '全品味', '散称', '批发', '86f47c35e2d4477d838e1280a949028b', '86f47c35e2d4477d838e1280a949028b', '王璐璐', '王璐璐', '2026-03-12 11:31:37', '2026-03-12 11:31:37', '377410', '38.85', '37.00');
INSERT INTO `bi_price_xx` VALUES ('008ff458-bc4d-4fa8-9612-1b50fcd86992', '去骨凤爪', '全品味', '72g', '批发', '86f47c35e2d4477d838e1280a949028b', '86f47c35e2d4477d838e1280a949028b', '王璐璐', '王璐璐', '2026-03-12 11:31:37', '2026-03-12 11:31:37', '377410', '10.40', '9.90');
INSERT INTO `bi_price_xx` VALUES ('d49be613-998a-4fd9-b629-ba01fc51876b', '去骨凤爪', '全品味', '138g', '批发', '86f47c35e2d4477d838e1280a949028b', '86f47c35e2d4477d838e1280a949028b', '王璐璐', '王璐璐', '2026-03-12 11:31:37', '2026-03-12 11:31:37', '377410', '17.33', '16.50');
INSERT INTO `bi_price_xx` VALUES ('8a2eab0f-f4f7-4192-9608-f2b6a7e17482', '老卤凤爪', '全品味', '95g', '批发', '86f47c35e2d4477d838e1280a949028b', '86f47c35e2d4477d838e1280a949028b', '王璐璐', '王璐璐', '2026-03-12 11:31:37', '2026-03-12 11:31:37', '377410', '10.40', '9.90');
INSERT INTO `bi_price_xx` VALUES ('069d4f5e-deb0-43fb-af81-d812f1f23465', '老卤鸭掌', '全品味', '95g', '批发', '86f47c35e2d4477d838e1280a949028b', '86f47c35e2d4477d838e1280a949028b', '王璐璐', '王璐璐', '2026-03-12 11:31:37', '2026-03-12 11:31:37', '377410', '10.40', '9.90');
INSERT INTO `bi_price_xx` VALUES ('e2a057f4-1890-4096-bb7b-80214b86d94d', '鸡肉豆堡', '全品味', '120g', '批发', '86f47c35e2d4477d838e1280a949028b', '86f47c35e2d4477d838e1280a949028b', '王璐璐', '王璐璐', '2026-03-12 11:31:37', '2026-03-12 11:31:37', '377410', '6.83', '6.50');
INSERT INTO `bi_price_xx` VALUES ('b07e83c2-7be9-49e2-b42e-12267010ed0e', '虎皮小鸡腿', '全品味', '80g', '批发', '86f47c35e2d4477d838e1280a949028b', '86f47c35e2d4477d838e1280a949028b', '王璐璐', '王璐璐', '2026-03-12 11:31:37', '2026-03-12 11:31:37', '377410', '8.40', '8.00');
config.py → code/config.py
浏览文件 @ ea335516
......@@ -7,8 +7,9 @@ import os
from typing import Optional
from dotenv import load_dotenv
# 加载 .env 文件
load_dotenv()
# 加载 .env 文件（使用绝对路径，避免因工作目录不同导致加载失败）
_env_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), '.env')
load_dotenv(dotenv_path=_env_path)
class Config:
"""应用配置类"""
......
core/db_handler.py → code/core/db_handler.py
浏览文件 @ ea335516
......@@ -6,8 +6,8 @@
import logging
import mysql.connector
from typing import List, Dict, Any
import os
from contextlib import contextmanager
from config import config
logger = logging.getLogger(__name__)
......@@ -17,11 +17,11 @@ class DatabaseHandler:
def __init__(self):
"""初始化数据库配置"""
self.db_config = {
'host': os.getenv('DB_HOST', 'localhost'),
'user': os.getenv('DB_USER', 'root'),
'password': os.getenv('DB_PASSWORD', ''),
'database': os.getenv('DB_NAME', 'clean_data'),
'port': int(os.getenv('DB_PORT', 3306)),
'host': config.DB_HOST,
'user': config.DB_USER,
'password': config.DB_PASSWORD,
'database': config.DB_NAME,
'port': config.DB_PORT,
'autocommit': False,
'connection_timeout': 10
}
......
code/core_py/1低价计算.py 0 → 100644
浏览文件 @ ea335516
import sys
import os
import pandas as pd
import mysql.connector
# 兼容直接运行（python core_py/1低价计算.py）和作为模块被 index.py 导入两种场景
if __name__ == "__main__":
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from config import config
def load_price_map_from_db() -> dict:
"""
从 market_bi.bi_price_xx 读取线下价盘数据，
返回匹配字典: { "产品系列|产品克重|渠道(大写)" -> low_price(float) }
"""
conn = mysql.connector.connect(
host=config.DB_HOST,
port=config.DB_PORT,
user=config.DB_USER,
password=config.DB_PASSWORD,
database="market_bi",
charset="utf8mb4",
)
try:
sql = "SELECT bi_product, pro_weight, channel_type, low_price FROM bi_price_xx"
df_p = pd.read_sql(sql, conn)
finally:
conn.close()
def _clean(s):
return "" if pd.isna(s) else str(s).strip().upper()
df_p["match_key"] = (
df_p["bi_product"].apply(_clean) + "|"
+ df_p["pro_weight"].apply(_clean) + "|"
+ df_p["channel_type"].apply(_clean)
)
df_p["low_price"] = pd.to_numeric(df_p["low_price"], errors="coerce")
return df_p.set_index("match_key")["low_price"].to_dict()
def transform(df_y: pd.DataFrame) -> pd.DataFrame:
"""
供 API 调用的低价计算入口。
接收大宽表 DataFrame（STANDARD_COLUMNS 列名），从数据库 market_bi.bi_price_xx
读取价盘基准，计算并回填以下三列后返回：
- 是否低价：低价 / 正常 / None（无法匹配或缺价格）
- 破价价差：低价时的价差（decimal），正常/无法匹配时为 None
- 低价整改状态：低价时置为 '未整改'，其余不改动
Args:
df_y: 大宽表 DataFrame，必须包含列：
产品系列、产品克重、渠道类型（稽查源提供）、产品价格
Returns:
pd.DataFrame: 更新了低价相关字段的 DataFrame（不修改原对象）
"""
df = df_y.copy()
price_map = load_price_map_from_db()
def _clean(s):
return "" if pd.isna(s) else str(s).strip().upper()
# 构建匹配键和数值价格（辅助列，最终会删除）
df["_series_c"] = df["产品系列"].apply(_clean)
df["_weight_c"] = df["产品克重"].apply(_clean)
df["_channel_c"] = df["渠道类型（稽查源提供）"].apply(_clean)
df["_match_key"] = df["_series_c"] + "|" + df["_weight_c"] + "|" + df["_channel_c"]
df["_price_num"] = pd.to_numeric(df["产品价格"], errors="coerce")
df["_p_low_price"] = df["_match_key"].map(price_map)
# 重置低价相关列
df["是否低价"] = None
df["破价价差"] = None
# 条件向量化计算，避免逐行循环
has_both = df["_price_num"].notna() & df["_p_low_price"].notna()
cond_low = has_both & (df["_price_num"] < df["_p_low_price"])
cond_normal = has_both & ~cond_low
df.loc[cond_low, "是否低价"] = "低价"
df.loc[cond_low, "破价价差"] = (
df.loc[cond_low, "_p_low_price"] - df.loc[cond_low, "_price_num"]
).round(2)
df["低价整改状态"] = df["低价整改状态"].astype(object)
df.loc[cond_low, "低价整改状态"] = "未整改"
df.loc[cond_normal, "是否低价"] = "正常"
df.loc[cond_normal, "破价价差"] = None
# 清除辅助列
df.drop(
columns=["_series_c", "_weight_c", "_channel_c", "_match_key", "_price_num", "_p_low_price"],
inplace=True,
)
return df
if __name__ == "__main__":
# ── 独立测试模式：读本地 Excel 大宽表 → 计算低价 → 输出结果文件 ──
from datetime import datetime
from dateutil.relativedelta import relativedelta
current_date = (datetime.now().replace(day=1) - relativedelta(months=1)).strftime("%Y-%m-01")
y_file = f"/王小卤/风控/代码-新/大日期{current_date}_2.xlsx"
output_file = f"/王小卤/风控/代码-新/低价大日期_2.xlsx"
print("正在读取稽查结果大宽表...")
df_y = pd.read_excel(y_file, sheet_name="合并后", dtype=str)
df_y.columns = df_y.columns.str.strip()
print("正在从数据库读取价盘并计算低价...")
df_result = transform(df_y)
df_result.to_excel(output_file, index=False)
print(f"✅ 处理完成！结果已保存至：{output_file}")
index.py → code/index.py
浏览文件 @ ea335516
......@@ -23,13 +23,109 @@ from utils.exceptions import DataCleaningException, DatabaseException
from utils.validators import validate_excel_url
from utils.response import BizCode, ok_resp, fail_resp
# 风控稽查大宽表：中文列名 → 数据库英文字段名
FENGKONG_COLUMN_MAP = {
"稽查日期": "audit_date",
"稽查来源": "source",
"大区": "region_name",
"战区": "district_name",
"经销商编码": "dealer_code",
"经销商名称": "dealer_name",
"勤策门店编码": "store_code",
"勤策门店名称": "store_name",
"客户经理工号": "f_emp_no",
"客户经理": "f_emp_name",
"勤策渠道大类": "qin_ce_type_large",
"稽核渠道（对N列清洗）": "jh_channel_type",
"城市": "city",
"渠道类型（稽查源提供）":"channel_type",
"产品系列": "series",
"产品口味": "taste",
"产品克重": "weight",
"产品价格": "price",
"是否低价": "low_price",
"破价价差": "low_price_diff",
"低价整改状态": "low_price_status",
"低价整改说明": "low_price_rectify",
"产品生产月份": "production_month",
"临期月份数": "near_month_num",
"临期状态": "near_month_status",
"新鲜度": "fresh_status",
"大日期整改状态": "large_date_status",
"大日期整改说明": "large_date_rectify",
}
# risk_audit_visit 各字段类型分组（用于入库前类型强制转换）
_FK_DECIMAL_COLS = {"price", "low_price_diff"}
_FK_INT_COLS = {"near_month_num"}
_FK_DATE_COLS = {"audit_date", "production_month"}
# varchar 字段最大长度限制（超长截断，防止 Data too long 报错）
_FK_VARCHAR_MAX = {
"source": 20, "region_name": 20, "district_name": 20,
"dealer_code": 10, "dealer_name": 100,
"store_code": 20, "store_name": 100,
"f_emp_no": 20, "f_emp_name": 100,
"qin_ce_type_large": 20, "jh_channel_type": 20,
"city": 30, "channel_type": 30,
"series": 20, "taste": 20, "weight": 20,
"low_price": 20, "low_price_status": 20, "low_price_rectify": 100,
"near_month_status": 20, "fresh_status": 20,
"large_date_status": 20, "large_date_rectify": 100,
}
def _coerce_fengkong_row(row: dict) -> dict:
"""
对已完成列名映射（英文 key）的行做类型强制转换，使其与 risk_audit_visit 字段类型完全匹配：
- decimal: 转 float，失败 → None
- int: 转 int，失败 → None
- date: 保留 'YYYY-MM-DD' 前10位，格式非法 → None
- varchar: 转字符串并按最大长度截断，空值 → None
"""
result = {}
for col, val in row.items():
# 统一空值处理
if val is None or (isinstance(val, str) and val.strip() == ''):
result[col] = None
continue
if col in _FK_DECIMAL_COLS:
try:
result[col] = float(val)
except (ValueError, TypeError):
result[col] = None
elif col in _FK_INT_COLS:
try:
result[col] = int(float(val))
except (ValueError, TypeError):
result[col] = None
elif col in _FK_DATE_COLS:
s = str(val)[:10]
try:
datetime.strptime(s, "%Y-%m-%d")
result[col] = s
except ValueError:
result[col] = None
else:
# varchar：转字符串，按最大长度截断
s = str(val).strip()
max_len = _FK_VARCHAR_MAX.get(col)
result[col] = s[:max_len] if max_len else s
return result
def _sanitize_nan(records: list) -> list:
"""将列表中每行 dict 里的 float NaN / Inf 替换为 None，确保 JSON 可序列化。"""
"""将列表中每行 dict 里的 float NaN / Inf 以及空字符串替换为 None，确保数据库写入兼容。"""
sanitized = []
for row in records:
sanitized.append({
k: (None if isinstance(v, float) and (math.isnan(v) or math.isinf(v)) else v)
k: (None if (isinstance(v, float) and (math.isnan(v) or math.isinf(v)))
or (isinstance(v, str) and v.strip() == '')
else v)
for k, v in row.items()
})
return sanitized
......@@ -68,7 +164,7 @@ class CleaningRequest(BaseModel):
class SavingRequest(BaseModel):
"""数据保存请求模型"""
task_id: str
table_name: str
table_name: Optional[str] = None # 风控稽查任务已预设表名，可不传
# ==================== 业务逻辑 ====================
......@@ -277,15 +373,30 @@ class DataCleaningService:
if task_id not in self.cleaned_data_cache:
raise DatabaseException(f"任务 {task_id} 的清洗数据不存在或已过期（超过30分钟）")
cleaned_data = self.cleaned_data_cache[task_id]['data']
cached = self.cleaned_data_cache[task_id]
cleaned_data = cached['data']
# 优先使用缓存中预设的表名（风控稽查任务已写死 risk_audit_visit）
target_table = cached.get('table_name') or table_name
if not target_table:
raise DatabaseException("未指定目标表名，请在请求中传入 table_name")
# 将中文列名映射为数据库英文字段名，并强制转换各字段类型（仅对 risk_audit_visit 生效）
if target_table == "risk_audit_visit":
cleaned_data = [
_coerce_fengkong_row(
{FENGKONG_COLUMN_MAP[k]: v for k, v in row.items() if k in FENGKONG_COLUMN_MAP}
)
for row in cleaned_data
]
# 保存到数据库
affected_rows = await self.db_handler.insert_data(
table_name,
target_table,
cleaned_data
)
logger.info(f"[{task_id}] 成功保存 {affected_rows} 行数据到 {table_name}")
logger.info(f"[{task_id}] 成功保存 {affected_rows} 行数据到 {target_table}")
# 清理缓存
del self.cleaned_data_cache[task_id]
......@@ -416,35 +527,50 @@ class DataCleaningService:
# ── 4. 合并为大宽表（内存，不写文件） ──────────────────
self.progress_manager.update_progress(
task_id, status="processing", progress=90, message="正在合并数据宽表..."
task_id, status="processing", progress=85, message="正在合并数据宽表..."
)
df_merged = pd.DataFrame(all_records, columns=STANDARD_COLUMNS)
merged_records = _sanitize_nan(
df_merged.where(pd.notna(df_merged), None).to_dict(orient="records")
logger.info(f"[{task_id}] 大宽表合并完成，共 {len(df_merged)} 条记录")
# ── 5. 低价计算（从数据库读取价盘，回填低价字段） ────────
self.progress_manager.update_progress(
task_id, status="processing", progress=93, message="正在执行低价计算..."
)
import importlib.util, pathlib
_lp_spec = importlib.util.spec_from_file_location(
"low_price_calc",
pathlib.Path(__file__).parent / "core_py" / "1低价计算.py",
)
_lp_mod = importlib.util.module_from_spec(_lp_spec)
_lp_spec.loader.exec_module(_lp_mod)
df_final = await asyncio.to_thread(_lp_mod.transform, df_merged)
final_records = _sanitize_nan(
df_final.where(pd.notna(df_final), None).to_dict(orient="records")
)
logger.info(f"[{task_id}] 大宽表合并完成，共 {len(merged_records)} 条记录")
logger.info(f"[{task_id}] 低价计算完成，共 {len(final_records)} 条记录")
# ── 5. 写入内存缓存 ──────────────────────────────────────
# ── 6. 写入内存缓存 ──────────────────────────────────────
self._evict_expired_cache()
self.cleaned_data_cache[task_id] = {
"data": merged_records,
"data": final_records,
"department": "风控稽查数据清洗",
"created_at": datetime.now(),
"row_count": len(merged_records),
"row_count": len(final_records),
"table_name": "risk_audit_visit",
}
self.progress_manager.update_progress(
task_id, status="completed", progress=100,
message=f"风控稽查数据清洗完成，共 {len(merged_records)} 条记录，等待前端确认",
processed_count=len(merged_records)
message=f"风控稽查数据清洗完成，共 {len(final_records)} 条记录，等待前端确认",
processed_count=len(final_records)
)
return {
"task_id": task_id,
"status": "completed",
"message": "风控稽查数据清洗成功",
"data_preview": merged_records[:5],
"total_rows": len(merged_records),
"data_preview": final_records[:5],
"total_rows": len(final_records),
}
except Exception as e:
......@@ -611,8 +737,8 @@ async def save_cleaned_data(request: SavingRequest):
Returns: { code, msg, data: { task_id, status, affected_rows } }
"""
try:
if not request.task_id or not request.table_name:
return fail_resp(BizCode.BAD_REQUEST, "参数不完整：task_id 和 table_name 均为必填")
if not request.task_id:
return fail_resp(BizCode.BAD_REQUEST, "参数不完整：task_id 为必填")
result = await service.save_cleaned_data(request.task_id, request.table_name)
......
core_py/1低价计算.py deleted 100644 → 0
浏览文件 @ fa6c4c14
import pandas as pd
from datetime import datetime
from dateutil.relativedelta import relativedelta
# 文件路径
# TODO: 配置稽查月份（默认1号）
current_date = (datetime.now().replace(day=1) - relativedelta(months=1)).strftime("%Y-%m-01")
y_file = f"/王小卤/风控/代码-新/大日期{current_date}_2.xlsx"
p_file = f"/王小卤/风控/代码-新//线下价盘表2601版.xlsx"
# 保存回原文件（建议先保存为新文件以防覆盖）
output_file = f"/王小卤/风控/代码-新//低价大日期_2.xlsx"
# 读取Y表（稽查结果表）
df_y = pd.read_excel(y_file,sheet_name='合并后', dtype=str) # 先以字符串读入避免格式问题，后续转数字
# 读取P表（价盘表）
df_p = pd.read_excel(p_file, dtype=str)
# 清理列名（去除前后空格等）
df_y.columns = df_y.columns.str.strip()
df_p.columns = df_p.columns.str.strip()
# 将关键字段转换为统一格式（去除空格、统一大小写等，便于匹配）
def clean_str(s):
if pd.isna(s):
return ""
return str(s).strip().upper()
# 对Y表的关键列清洗
df_y['产品系列_clean'] = df_y.iloc[:, 14].apply(clean_str) # O列：产品系列
df_y['产品克重_clean'] = df_y.iloc[:, 16].apply(clean_str) # Q列：产品克重
df_y['渠道类型_clean'] = df_y.iloc[:, 13].apply(clean_str) # N列：渠道类型（稽查源提供）
# 对P表的关键列清洗
df_p['产品系统_clean'] = df_p.iloc[:, 0].apply(clean_str) # A列：产品系统
df_p['产品克重_p_clean'] = df_p.iloc[:, 2].apply(clean_str) # C列：产品克重
df_p['渠道_p_clean'] = df_p.iloc[:, 3].apply(clean_str) # D列：渠道
# 将价格列转为数值类型（注意处理非数字情况）
df_y['产品价格_num'] = pd.to_numeric(df_y.iloc[:, 17], errors='coerce') # R列：产品价格
df_p['低价_num'] = pd.to_numeric(df_p.iloc[:, 4], errors='coerce') # E列：低价
# 构建P表的唯一键（产品系统 + 产品克重 + 渠道）
df_p['match_key'] = df_p['产品系统_clean'] + '|' + df_p['产品克重_p_clean'] + '|' + df_p['渠道_p_clean']
# 构建Y表的匹配键（产品系列 + 产品克重 + 渠道类型）
df_y['match_key'] = df_y['产品系列_clean'] + '|' + df_y['产品克重_clean'] + '|' + df_y['渠道类型_clean']
# 将P表转为字典：key -> 低价
price_map = df_p.set_index('match_key')['低价_num'].to_dict()
# 初始化Y表的目标列（S: 是否低价, T: 破价价差）
df_y['是否低价'] = '正常' # 默认值
df_y['破价价差'] = None
# 遍历Y表每一行进行匹配和判断
for idx, row in df_y.iterrows():
key = row['match_key']
y_price = row['产品价格_num']
p_low_price = price_map.get(key, None)
if pd.notna(y_price) and pd.notna(p_low_price):
if y_price < p_low_price:
df_y.at[idx, '是否低价'] = '低价'
df_y.at[idx, '破价价差'] = round(p_low_price - y_price, 2)
df_y.at[idx, '低价整改状态'] = '未整改'
else:
df_y.at[idx, '是否低价'] = '正常'
df_y.at[idx, '破价价差'] = None
else:
# 无法匹配或价格缺失，保留默认或标记
df_y.at[idx, '是否低价'] = None
df_y.at[idx, '破价价差'] = None
# 只保留原始列（不保留清洗用的辅助列）
original_columns = df_y.columns.tolist()
output_columns = [col for col in original_columns if not col.endswith('_clean') and col not in ['产品价格_num', 'match_key']]
df_y[output_columns].to_excel(output_file, index=False)
print(f"处理完成！结果已保存至：{output_file}")
\ No newline at end of file
Markdown 格式
0% 或
您添加了 0 人 到此讨论。请谨慎行事。
请先完成此评论的编辑！
请 注册 或者 后发表评论