Skip to content

W
wangxiaolu-link-python-clean-data
提交 ea335516 authored 作者: lidongxu's avatar lidongxu
浏览文件
操作

包一层文件夹

上级 fa6c4c14
隐藏空白字符变更
内嵌 并排
正在显示 包含 363 行增加 和 109 行删除
code/bi_price_xx.sql 0 → 100644
浏览文件 @ ea335516
/*
Navicat MySQL Data Transfer
Source Server : t100_production
Source Server Version : 50744
Source Host : rm-2ze28qp55mrm34g8bbo.mysql.rds.aliyuncs.com:3306
Source Database : market_bi
Target Server Type : MYSQL
Target Server Version : 50744
File Encoding : 65001
Date: 2026-03-12 11:37:31
*/
SET FOREIGN_KEY_CHECKS=0;
-- ----------------------------
-- Table structure for bi_price_xx
-- ----------------------------
DROP TABLE IF EXISTS `bi_price_xx`;
CREATE TABLE `bi_price_xx` (
`id` varchar(255) COLLATE utf8mb4_bin DEFAULT NULL COMMENT '主键',
`bi_product` varchar(255) COLLATE utf8mb4_bin DEFAULT NULL COMMENT '产品系统',
`prd_name` varchar(255) COLLATE utf8mb4_bin DEFAULT NULL COMMENT '口味',
`pro_weight` varchar(255) COLLATE utf8mb4_bin DEFAULT NULL COMMENT '产品克重',
`channel_type` varchar(255) COLLATE utf8mb4_bin DEFAULT NULL COMMENT '渠道',
`creator` varchar(255) COLLATE utf8mb4_bin DEFAULT NULL COMMENT '提交人',
`modifier` varchar(255) COLLATE utf8mb4_bin DEFAULT NULL COMMENT '修改人',
`creator_nickname` varchar(255) COLLATE utf8mb4_bin DEFAULT NULL COMMENT '提交人昵称',
`modifier_nickname` varchar(255) COLLATE utf8mb4_bin DEFAULT NULL COMMENT '修改人昵称',
`create_time` datetime DEFAULT NULL COMMENT '提交时间',
`modify_time` datetime DEFAULT NULL COMMENT '修改时间',
`qbi_system_upload_id` bigint(30) DEFAULT NULL COMMENT '上传批次主键',
`low_price` decimal(30,2) DEFAULT NULL COMMENT '低价',
`normal_price` decimal(30,2) DEFAULT NULL COMMENT '零售价'
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin COMMENT='线下价盘表';
-- ----------------------------
-- Records of bi_price_xx
-- ----------------------------
INSERT INTO `bi_price_xx` VALUES ('7ac70b27-59d8-413c-81e4-d11d3e753ca2', '虎皮凤爪', '全品味', '105g', 'KA', '86f47c35e2d4477d838e1280a949028b', '86f47c35e2d4477d838e1280a949028b', '王璐璐', '王璐璐', '2026-03-12 11:31:37', '2026-03-12 11:31:37', '377410', '14.90', '21.89');
INSERT INTO `bi_price_xx` VALUES ('3ebea3f4-4e40-4088-aacb-d5d7b3194d82', '虎皮凤爪', '全品味', '210g', 'KA', '86f47c35e2d4477d838e1280a949028b', '86f47c35e2d4477d838e1280a949028b', '王璐璐', '王璐璐', '2026-03-12 11:31:37', '2026-03-12 11:31:37', '377410', '26.50', '38.39');
INSERT INTO `bi_price_xx` VALUES ('344df49c-597d-4826-b659-2fd1638edc45', '虎皮凤爪', '全品味', '散称', 'KA', '86f47c35e2d4477d838e1280a949028b', '86f47c35e2d4477d838e1280a949028b', '王璐璐', '王璐璐', '2026-03-12 11:31:37', '2026-03-12 11:31:37', '377410', '45.80', '54.78');
INSERT INTO `bi_price_xx` VALUES ('21f14713-eb58-4090-8df0-aa8a5b98f3e8', '去骨凤爪', '全品味', '72g', 'KA', '86f47c35e2d4477d838e1280a949028b', '86f47c35e2d4477d838e1280a949028b', '王璐璐', '王璐璐', '2026-03-12 11:31:37', '2026-03-12 11:31:37', '377410', '14.90', '21.89');
INSERT INTO `bi_price_xx` VALUES ('7d222a14-8191-4f88-9fec-2fa0e21c27cc', '去骨凤爪', '全品味', '138g', 'KA', '86f47c35e2d4477d838e1280a949028b', '86f47c35e2d4477d838e1280a949028b', '王璐璐', '王璐璐', '2026-03-12 11:31:37', '2026-03-12 11:31:37', '377410', '26.50', '38.39');
INSERT INTO `bi_price_xx` VALUES ('3d2ced09-1e88-46ef-87a3-e2368e5aaff1', '脆笋去骨', '全品味', '散称', 'KA', '86f47c35e2d4477d838e1280a949028b', '86f47c35e2d4477d838e1280a949028b', '王璐璐', '王璐璐', '2026-03-12 11:31:37', '2026-03-12 11:31:37', '377410', '45.80', '50.38');
INSERT INTO `bi_price_xx` VALUES ('bcc8e320-0db0-4535-9fd0-3e79585881c8', '老卤凤爪', '全品味', '95g', 'KA', '86f47c35e2d4477d838e1280a949028b', '86f47c35e2d4477d838e1280a949028b', '王璐璐', '王璐璐', '2026-03-12 11:31:37', '2026-03-12 11:31:37', '377410', '14.90', '21.89');
INSERT INTO `bi_price_xx` VALUES ('f795e572-7b8a-41f9-b2a4-678803019224', '老卤鸭掌', '全品味', '95g', 'KA', '86f47c35e2d4477d838e1280a949028b', '86f47c35e2d4477d838e1280a949028b', '王璐璐', '王璐璐', '2026-03-12 11:31:37', '2026-03-12 11:31:37', '377410', '14.90', '21.89');
INSERT INTO `bi_price_xx` VALUES ('de4bcae9-6ace-43b1-b88f-7c70ca674b64', '鸡肉豆堡', '全品味', '120g', 'KA', '86f47c35e2d4477d838e1280a949028b', '86f47c35e2d4477d838e1280a949028b', '王璐璐', '王璐璐', '2026-03-12 11:31:37', '2026-03-12 11:31:37', '377410', '9.90', '14.19');
INSERT INTO `bi_price_xx` VALUES ('4949ad7f-8f90-4bf9-8848-e862b210e3e0', '虎皮小鸡腿', '全品味', '80g', 'KA', '86f47c35e2d4477d838e1280a949028b', '86f47c35e2d4477d838e1280a949028b', '王璐璐', '王璐璐', '2026-03-12 11:31:37', '2026-03-12 11:31:37', '377410', '12.80', '18.48');
INSERT INTO `bi_price_xx` VALUES ('4286df2e-dcf4-4dd5-a25e-c26d8a3c8829', '虎皮凤爪', '全品味', '105g', 'BC', '86f47c35e2d4477d838e1280a949028b', '86f47c35e2d4477d838e1280a949028b', '王璐璐', '王璐璐', '2026-03-12 11:31:37', '2026-03-12 11:31:37', '377410', '14.90', '21.89');
INSERT INTO `bi_price_xx` VALUES ('e8b863b9-ce10-4a33-b142-6019c62aee93', '虎皮凤爪', '全品味', '210g', 'BC', '86f47c35e2d4477d838e1280a949028b', '86f47c35e2d4477d838e1280a949028b', '王璐璐', '王璐璐', '2026-03-12 11:31:37', '2026-03-12 11:31:37', '377410', '26.50', '38.39');
INSERT INTO `bi_price_xx` VALUES ('07ddcf94-619e-468a-9ad7-67b1835c5898', '虎皮凤爪', '全品味', '68g', 'BC', '86f47c35e2d4477d838e1280a949028b', '86f47c35e2d4477d838e1280a949028b', '王璐璐', '王璐璐', '2026-03-12 11:31:37', '2026-03-12 11:31:37', '377410', '9.80', '14.19');
INSERT INTO `bi_price_xx` VALUES ('0a193d95-e8ec-4e12-abe0-5dc465b1e1bd', '虎皮凤爪', '全品味', '25g', 'BC', '86f47c35e2d4477d838e1280a949028b', '86f47c35e2d4477d838e1280a949028b', '王璐璐', '王璐璐', '2026-03-12 11:31:37', '2026-03-12 11:31:37', '377410', '3.90', '5.39');
INSERT INTO `bi_price_xx` VALUES ('347aa860-eadf-4ae0-aa31-7a64f15322b3', '虎皮凤爪', '全品味', '散称', 'BC', '86f47c35e2d4477d838e1280a949028b', '86f47c35e2d4477d838e1280a949028b', '王璐璐', '王璐璐', '2026-03-12 11:31:37', '2026-03-12 11:31:37', '377410', '45.80', '54.78');
INSERT INTO `bi_price_xx` VALUES ('24e265c7-d526-4b30-a981-2eae56ac2227', '去骨凤爪', '全品味', '72g', 'BC', '86f47c35e2d4477d838e1280a949028b', '86f47c35e2d4477d838e1280a949028b', '王璐璐', '王璐璐', '2026-03-12 11:31:37', '2026-03-12 11:31:37', '377410', '14.90', '21.89');
INSERT INTO `bi_price_xx` VALUES ('71a70334-e06a-43a0-913e-e81d1eafcf94', '老卤凤爪', '全品味', '95g', 'BC', '86f47c35e2d4477d838e1280a949028b', '86f47c35e2d4477d838e1280a949028b', '王璐璐', '王璐璐', '2026-03-12 11:31:37', '2026-03-12 11:31:37', '377410', '14.90', '21.89');
INSERT INTO `bi_price_xx` VALUES ('b9686a81-9771-4c4b-8c4f-faf0b5fdf71f', '老卤鸭掌', '全品味', '95g', 'BC', '86f47c35e2d4477d838e1280a949028b', '86f47c35e2d4477d838e1280a949028b', '王璐璐', '王璐璐', '2026-03-12 11:31:37', '2026-03-12 11:31:37', '377410', '14.90', '21.89');
INSERT INTO `bi_price_xx` VALUES ('350cc768-1407-40ec-b9c4-4a336cd89118', '鸡肉豆堡', '全品味', '120g', 'BC', '86f47c35e2d4477d838e1280a949028b', '86f47c35e2d4477d838e1280a949028b', '王璐璐', '王璐璐', '2026-03-12 11:31:37', '2026-03-12 11:31:37', '377410', '9.90', '14.19');
INSERT INTO `bi_price_xx` VALUES ('5c67fa66-799b-43bc-8263-b84291ec7c44', '鸡肉豆堡', '全品味', '散称', 'BC', '86f47c35e2d4477d838e1280a949028b', '86f47c35e2d4477d838e1280a949028b', '王璐璐', '王璐璐', '2026-03-12 11:31:37', '2026-03-12 11:31:37', '377410', '34.90', '38.39');
INSERT INTO `bi_price_xx` VALUES ('09084915-3a6a-4339-a8b1-f26ca803aec6', '虎皮小鸡腿', '全品味', '80g', 'BC', '86f47c35e2d4477d838e1280a949028b', '86f47c35e2d4477d838e1280a949028b', '王璐璐', '王璐璐', '2026-03-12 11:31:37', '2026-03-12 11:31:37', '377410', '12.80', '18.48');
INSERT INTO `bi_price_xx` VALUES ('54b65a1e-41b2-4d86-9895-64eb03f33ed0', '虎皮凤爪', '全品味', '105g', 'CVS', '86f47c35e2d4477d838e1280a949028b', '86f47c35e2d4477d838e1280a949028b', '王璐璐', '王璐璐', '2026-03-12 11:31:37', '2026-03-12 11:31:37', '377410', '14.90', '21.89');
INSERT INTO `bi_price_xx` VALUES ('44b67fcf-42ac-4b93-83e0-da427b013737', '虎皮凤爪', '全品味', '210g', 'CVS', '86f47c35e2d4477d838e1280a949028b', '86f47c35e2d4477d838e1280a949028b', '王璐璐', '王璐璐', '2026-03-12 11:31:37', '2026-03-12 11:31:37', '377410', '29.50', '38.39');
INSERT INTO `bi_price_xx` VALUES ('d5dfc92a-a075-4277-938b-269f103aa76f', '虎皮凤爪', '全品味', '68g', 'CVS', '86f47c35e2d4477d838e1280a949028b', '86f47c35e2d4477d838e1280a949028b', '王璐璐', '王璐璐', '2026-03-12 11:31:37', '2026-03-12 11:31:37', '377410', '10.90', '14.19');
INSERT INTO `bi_price_xx` VALUES ('ce9500ff-6671-4514-8a40-6a84c79ab1a3', '虎皮凤爪', '全品味', '25g', 'CVS', '86f47c35e2d4477d838e1280a949028b', '86f47c35e2d4477d838e1280a949028b', '王璐璐', '王璐璐', '2026-03-12 11:31:37', '2026-03-12 11:31:37', '377410', '3.90', '5.39');
INSERT INTO `bi_price_xx` VALUES ('0a34f1bc-7e5e-4cf3-a059-da9a29d0bad8', '去骨凤爪', '全品味', '72g', 'CVS', '86f47c35e2d4477d838e1280a949028b', '86f47c35e2d4477d838e1280a949028b', '王璐璐', '王璐璐', '2026-03-12 11:31:37', '2026-03-12 11:31:37', '377410', '14.90', '21.89');
INSERT INTO `bi_price_xx` VALUES ('0bf423ef-5275-43ce-80b4-dac3662b21c2', '去骨凤爪', '全品味', '138g', 'CVS', '86f47c35e2d4477d838e1280a949028b', '86f47c35e2d4477d838e1280a949028b', '王璐璐', '王璐璐', '2026-03-12 11:31:37', '2026-03-12 11:31:37', '377410', '29.50', '38.39');
INSERT INTO `bi_price_xx` VALUES ('0ebf0fe1-25cd-4b1c-bb16-698d714a90f6', '老卤凤爪', '全品味', '95g', 'CVS', '86f47c35e2d4477d838e1280a949028b', '86f47c35e2d4477d838e1280a949028b', '王璐璐', '王璐璐', '2026-03-12 11:31:37', '2026-03-12 11:31:37', '377410', '14.90', '21.89');
INSERT INTO `bi_price_xx` VALUES ('3456b8c9-772f-4c8b-a2a1-a2562273839f', '老卤鸭掌', '全品味', '95g', 'CVS', '86f47c35e2d4477d838e1280a949028b', '86f47c35e2d4477d838e1280a949028b', '王璐璐', '王璐璐', '2026-03-12 11:31:37', '2026-03-12 11:31:37', '377410', '14.90', '21.89');
INSERT INTO `bi_price_xx` VALUES ('55957f41-d285-4dac-b661-b1c6f7faf4c1', '鸡肉豆堡', '全品味', '120g', 'CVS', '86f47c35e2d4477d838e1280a949028b', '86f47c35e2d4477d838e1280a949028b', '王璐璐', '王璐璐', '2026-03-12 11:31:37', '2026-03-12 11:31:37', '377410', '9.90', '14.19');
INSERT INTO `bi_price_xx` VALUES ('44274c3b-697b-401b-9670-b663a235cbf9', '虎皮小鸡腿', '全品味', '80g', 'CVS', '86f47c35e2d4477d838e1280a949028b', '86f47c35e2d4477d838e1280a949028b', '王璐璐', '王璐璐', '2026-03-12 11:31:37', '2026-03-12 11:31:37', '377410', '12.80', '18.48');
INSERT INTO `bi_price_xx` VALUES ('fb961ee5-3d1c-406b-81f0-68f73a24e82a', '虎皮凤爪', '全品味', '68g', '零食', '86f47c35e2d4477d838e1280a949028b', '86f47c35e2d4477d838e1280a949028b', '王璐璐', '王璐璐', '2026-03-12 11:31:37', '2026-03-12 11:31:37', '377410', '8.80', '14.19');
INSERT INTO `bi_price_xx` VALUES ('e400d8e8-19ce-41dd-b02e-391933d76d43', '虎皮凤爪', '全品味', '散称', '零食', '86f47c35e2d4477d838e1280a949028b', '86f47c35e2d4477d838e1280a949028b', '王璐璐', '王璐璐', '2026-03-12 11:31:37', '2026-03-12 11:31:37', '377410', '45.80', '54.78');
INSERT INTO `bi_price_xx` VALUES ('edacdcb9-6450-4965-870c-979f5aa0fdf3', '脆笋去骨', '全品味', '散称', '零食', '86f47c35e2d4477d838e1280a949028b', '86f47c35e2d4477d838e1280a949028b', '王璐璐', '王璐璐', '2026-03-12 11:31:37', '2026-03-12 11:31:37', '377410', '45.80', '50.38');
INSERT INTO `bi_price_xx` VALUES ('d75c4276-56e6-4296-b389-8ea2ff386d48', '鸡肉豆堡', '全品味', '散称', '零食', '86f47c35e2d4477d838e1280a949028b', '86f47c35e2d4477d838e1280a949028b', '王璐璐', '王璐璐', '2026-03-12 11:31:37', '2026-03-12 11:31:37', '377410', '34.90', '38.39');
INSERT INTO `bi_price_xx` VALUES ('b748e150-82ad-447d-89f7-7a5b4efc5b25', '虎皮小鸡腿', '全品味', '散称', '零食', '86f47c35e2d4477d838e1280a949028b', '86f47c35e2d4477d838e1280a949028b', '王璐璐', '王璐璐', '2026-03-12 11:31:37', '2026-03-12 11:31:37', '377410', '35.80', '39.38');
INSERT INTO `bi_price_xx` VALUES ('a8854d47-e1a3-4673-afb5-9b7d0fa8d183', '虎皮凤爪', '全品味', '105g', '批发', '86f47c35e2d4477d838e1280a949028b', '86f47c35e2d4477d838e1280a949028b', '王璐璐', '王璐璐', '2026-03-12 11:31:37', '2026-03-12 11:31:37', '377410', '10.40', '9.90');
INSERT INTO `bi_price_xx` VALUES ('b93e7fef-8ecf-4434-9441-da61b839069b', '虎皮凤爪', '全品味', '210g', '批发', '86f47c35e2d4477d838e1280a949028b', '86f47c35e2d4477d838e1280a949028b', '王璐璐', '王璐璐', '2026-03-12 11:31:37', '2026-03-12 11:31:37', '377410', '19.11', '18.20');
INSERT INTO `bi_price_xx` VALUES ('cc8f1359-2afc-464c-8f0f-62e985e99fb7', '虎皮凤爪', '全品味', '68g', '批发', '86f47c35e2d4477d838e1280a949028b', '86f47c35e2d4477d838e1280a949028b', '王璐璐', '王璐璐', '2026-03-12 11:31:37', '2026-03-12 11:31:37', '377410', '6.83', '6.50');
INSERT INTO `bi_price_xx` VALUES ('5711f296-0451-48c6-bcf7-efdb68d5867b', '虎皮凤爪', '全品味', '25g', '批发', '86f47c35e2d4477d838e1280a949028b', '86f47c35e2d4477d838e1280a949028b', '王璐璐', '王璐璐', '2026-03-12 11:31:37', '2026-03-12 11:31:37', '377410', '2.63', '2.50');
INSERT INTO `bi_price_xx` VALUES ('aab9599c-2a33-437d-8ef8-890ed1bec685', '虎皮凤爪', '全品味', '散称', '批发', '86f47c35e2d4477d838e1280a949028b', '86f47c35e2d4477d838e1280a949028b', '王璐璐', '王璐璐', '2026-03-12 11:31:37', '2026-03-12 11:31:37', '377410', '38.85', '37.00');
INSERT INTO `bi_price_xx` VALUES ('008ff458-bc4d-4fa8-9612-1b50fcd86992', '去骨凤爪', '全品味', '72g', '批发', '86f47c35e2d4477d838e1280a949028b', '86f47c35e2d4477d838e1280a949028b', '王璐璐', '王璐璐', '2026-03-12 11:31:37', '2026-03-12 11:31:37', '377410', '10.40', '9.90');
INSERT INTO `bi_price_xx` VALUES ('d49be613-998a-4fd9-b629-ba01fc51876b', '去骨凤爪', '全品味', '138g', '批发', '86f47c35e2d4477d838e1280a949028b', '86f47c35e2d4477d838e1280a949028b', '王璐璐', '王璐璐', '2026-03-12 11:31:37', '2026-03-12 11:31:37', '377410', '17.33', '16.50');
INSERT INTO `bi_price_xx` VALUES ('8a2eab0f-f4f7-4192-9608-f2b6a7e17482', '老卤凤爪', '全品味', '95g', '批发', '86f47c35e2d4477d838e1280a949028b', '86f47c35e2d4477d838e1280a949028b', '王璐璐', '王璐璐', '2026-03-12 11:31:37', '2026-03-12 11:31:37', '377410', '10.40', '9.90');
INSERT INTO `bi_price_xx` VALUES ('069d4f5e-deb0-43fb-af81-d812f1f23465', '老卤鸭掌', '全品味', '95g', '批发', '86f47c35e2d4477d838e1280a949028b', '86f47c35e2d4477d838e1280a949028b', '王璐璐', '王璐璐', '2026-03-12 11:31:37', '2026-03-12 11:31:37', '377410', '10.40', '9.90');
INSERT INTO `bi_price_xx` VALUES ('e2a057f4-1890-4096-bb7b-80214b86d94d', '鸡肉豆堡', '全品味', '120g', '批发', '86f47c35e2d4477d838e1280a949028b', '86f47c35e2d4477d838e1280a949028b', '王璐璐', '王璐璐', '2026-03-12 11:31:37', '2026-03-12 11:31:37', '377410', '6.83', '6.50');
INSERT INTO `bi_price_xx` VALUES ('b07e83c2-7be9-49e2-b42e-12267010ed0e', '虎皮小鸡腿', '全品味', '80g', '批发', '86f47c35e2d4477d838e1280a949028b', '86f47c35e2d4477d838e1280a949028b', '王璐璐', '王璐璐', '2026-03-12 11:31:37', '2026-03-12 11:31:37', '377410', '8.40', '8.00');
config.py → code/config.py
浏览文件 @ ea335516
...@@ -7,8 +7,9 @@ import os ...@@ -7,8 +7,9 @@ import os
from typing import Optional from typing import Optional
from dotenv import load_dotenv from dotenv import load_dotenv
# 加载 .env 文件 # 加载 .env 文件（使用绝对路径，避免因工作目录不同导致加载失败）
load_dotenv() _env_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), '.env')
load_dotenv(dotenv_path=_env_path)
class Config: class Config:
"""应用配置类""" """应用配置类"""
......
core/db_handler.py → code/core/db_handler.py
浏览文件 @ ea335516
...@@ -6,8 +6,8 @@ ...@@ -6,8 +6,8 @@
import logging import logging
import mysql.connector import mysql.connector
from typing import List, Dict, Any from typing import List, Dict, Any
import os
from contextlib import contextmanager from contextlib import contextmanager
from config import config
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
...@@ -17,11 +17,11 @@ class DatabaseHandler: ...@@ -17,11 +17,11 @@ class DatabaseHandler:
def __init__(self): def __init__(self):
"""初始化数据库配置""" """初始化数据库配置"""
self.db_config = { self.db_config = {
'host': os.getenv('DB_HOST', 'localhost'), 'host': config.DB_HOST,
'user': os.getenv('DB_USER', 'root'), 'user': config.DB_USER,
'password': os.getenv('DB_PASSWORD', ''), 'password': config.DB_PASSWORD,
'database': os.getenv('DB_NAME', 'clean_data'), 'database': config.DB_NAME,
'port': int(os.getenv('DB_PORT', 3306)), 'port': config.DB_PORT,
'autocommit': False, 'autocommit': False,
'connection_timeout': 10 'connection_timeout': 10
} }
......
code/core_py/1低价计算.py 0 → 100644
浏览文件 @ ea335516
import sys
import os
import pandas as pd
import mysql.connector
# 兼容直接运行（python core_py/1低价计算.py）和作为模块被 index.py 导入两种场景
if __name__ == "__main__":
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from config import config
def load_price_map_from_db() -> dict:
"""
从 market_bi.bi_price_xx 读取线下价盘数据，
返回匹配字典: { "产品系列|产品克重|渠道(大写)" -> low_price(float) }
"""
conn = mysql.connector.connect(
host=config.DB_HOST,
port=config.DB_PORT,
user=config.DB_USER,
password=config.DB_PASSWORD,
database="market_bi",
charset="utf8mb4",
)
try:
sql = "SELECT bi_product, pro_weight, channel_type, low_price FROM bi_price_xx"
df_p = pd.read_sql(sql, conn)
finally:
conn.close()
def _clean(s):
return "" if pd.isna(s) else str(s).strip().upper()
df_p["match_key"] = (
df_p["bi_product"].apply(_clean) + "|"
+ df_p["pro_weight"].apply(_clean) + "|"
+ df_p["channel_type"].apply(_clean)
)
df_p["low_price"] = pd.to_numeric(df_p["low_price"], errors="coerce")
return df_p.set_index("match_key")["low_price"].to_dict()
def transform(df_y: pd.DataFrame) -> pd.DataFrame:
"""
供 API 调用的低价计算入口。
接收大宽表 DataFrame（STANDARD_COLUMNS 列名），从数据库 market_bi.bi_price_xx
读取价盘基准，计算并回填以下三列后返回：
- 是否低价：低价 / 正常 / None（无法匹配或缺价格）
- 破价价差：低价时的价差（decimal），正常/无法匹配时为 None
- 低价整改状态：低价时置为 '未整改'，其余不改动
Args:
df_y: 大宽表 DataFrame，必须包含列：
产品系列、产品克重、渠道类型（稽查源提供）、产品价格
Returns:
pd.DataFrame: 更新了低价相关字段的 DataFrame（不修改原对象）
"""
df = df_y.copy()
price_map = load_price_map_from_db()
def _clean(s):
return "" if pd.isna(s) else str(s).strip().upper()
# 构建匹配键和数值价格（辅助列，最终会删除）
df["_series_c"] = df["产品系列"].apply(_clean)
df["_weight_c"] = df["产品克重"].apply(_clean)
df["_channel_c"] = df["渠道类型（稽查源提供）"].apply(_clean)
df["_match_key"] = df["_series_c"] + "|" + df["_weight_c"] + "|" + df["_channel_c"]
df["_price_num"] = pd.to_numeric(df["产品价格"], errors="coerce")
df["_p_low_price"] = df["_match_key"].map(price_map)
# 重置低价相关列
df["是否低价"] = None
df["破价价差"] = None
# 条件向量化计算，避免逐行循环
has_both = df["_price_num"].notna() & df["_p_low_price"].notna()
cond_low = has_both & (df["_price_num"] < df["_p_low_price"])
cond_normal = has_both & ~cond_low
df.loc[cond_low, "是否低价"] = "低价"
df.loc[cond_low, "破价价差"] = (
df.loc[cond_low, "_p_low_price"] - df.loc[cond_low, "_price_num"]
).round(2)
df["低价整改状态"] = df["低价整改状态"].astype(object)
df.loc[cond_low, "低价整改状态"] = "未整改"
df.loc[cond_normal, "是否低价"] = "正常"
df.loc[cond_normal, "破价价差"] = None
# 清除辅助列
df.drop(
columns=["_series_c", "_weight_c", "_channel_c", "_match_key", "_price_num", "_p_low_price"],
inplace=True,
)
return df
if __name__ == "__main__":
# ── 独立测试模式：读本地 Excel 大宽表 → 计算低价 → 输出结果文件 ──
from datetime import datetime
from dateutil.relativedelta import relativedelta
current_date = (datetime.now().replace(day=1) - relativedelta(months=1)).strftime("%Y-%m-01")
y_file = f"/王小卤/风控/代码-新/大日期{current_date}_2.xlsx"
output_file = f"/王小卤/风控/代码-新/低价大日期_2.xlsx"
print("正在读取稽查结果大宽表...")
df_y = pd.read_excel(y_file, sheet_name="合并后", dtype=str)
df_y.columns = df_y.columns.str.strip()
print("正在从数据库读取价盘并计算低价...")
df_result = transform(df_y)
df_result.to_excel(output_file, index=False)
print(f"✅ 处理完成！结果已保存至：{output_file}")
index.py → code/index.py
浏览文件 @ ea335516
...@@ -23,13 +23,109 @@ from utils.exceptions import DataCleaningException, DatabaseException ...@@ -23,13 +23,109 @@ from utils.exceptions import DataCleaningException, DatabaseException
from utils.validators import validate_excel_url from utils.validators import validate_excel_url
from utils.response import BizCode, ok_resp, fail_resp from utils.response import BizCode, ok_resp, fail_resp
# 风控稽查大宽表：中文列名 → 数据库英文字段名
FENGKONG_COLUMN_MAP = {
"稽查日期": "audit_date",
"稽查来源": "source",
"大区": "region_name",
"战区": "district_name",
"经销商编码": "dealer_code",
"经销商名称": "dealer_name",
"勤策门店编码": "store_code",
"勤策门店名称": "store_name",
"客户经理工号": "f_emp_no",
"客户经理": "f_emp_name",
"勤策渠道大类": "qin_ce_type_large",
"稽核渠道（对N列清洗）": "jh_channel_type",
"城市": "city",
"渠道类型（稽查源提供）":"channel_type",
"产品系列": "series",
"产品口味": "taste",
"产品克重": "weight",
"产品价格": "price",
"是否低价": "low_price",
"破价价差": "low_price_diff",
"低价整改状态": "low_price_status",
"低价整改说明": "low_price_rectify",
"产品生产月份": "production_month",
"临期月份数": "near_month_num",
"临期状态": "near_month_status",
"新鲜度": "fresh_status",
"大日期整改状态": "large_date_status",
"大日期整改说明": "large_date_rectify",
}
# risk_audit_visit 各字段类型分组（用于入库前类型强制转换）
_FK_DECIMAL_COLS = {"price", "low_price_diff"}
_FK_INT_COLS = {"near_month_num"}
_FK_DATE_COLS = {"audit_date", "production_month"}
# varchar 字段最大长度限制（超长截断，防止 Data too long 报错）
_FK_VARCHAR_MAX = {
"source": 20, "region_name": 20, "district_name": 20,
"dealer_code": 10, "dealer_name": 100,
"store_code": 20, "store_name": 100,
"f_emp_no": 20, "f_emp_name": 100,
"qin_ce_type_large": 20, "jh_channel_type": 20,
"city": 30, "channel_type": 30,
"series": 20, "taste": 20, "weight": 20,
"low_price": 20, "low_price_status": 20, "low_price_rectify": 100,
"near_month_status": 20, "fresh_status": 20,
"large_date_status": 20, "large_date_rectify": 100,
}
def _coerce_fengkong_row(row: dict) -> dict:
"""
对已完成列名映射（英文 key）的行做类型强制转换，使其与 risk_audit_visit 字段类型完全匹配：
- decimal: 转 float，失败 → None
- int: 转 int，失败 → None
- date: 保留 'YYYY-MM-DD' 前10位，格式非法 → None
- varchar: 转字符串并按最大长度截断，空值 → None
"""
result = {}
for col, val in row.items():
# 统一空值处理
if val is None or (isinstance(val, str) and val.strip() == ''):
result[col] = None
continue
if col in _FK_DECIMAL_COLS:
try:
result[col] = float(val)
except (ValueError, TypeError):
result[col] = None
elif col in _FK_INT_COLS:
try:
result[col] = int(float(val))
except (ValueError, TypeError):
result[col] = None
elif col in _FK_DATE_COLS:
s = str(val)[:10]
try:
datetime.strptime(s, "%Y-%m-%d")
result[col] = s
except ValueError:
result[col] = None
else:
# varchar：转字符串，按最大长度截断
s = str(val).strip()
max_len = _FK_VARCHAR_MAX.get(col)
result[col] = s[:max_len] if max_len else s
return result
def _sanitize_nan(records: list) -> list: def _sanitize_nan(records: list) -> list:
"""将列表中每行 dict 里的 float NaN / Inf 替换为 None，确保 JSON 可序列化。""" """将列表中每行 dict 里的 float NaN / Inf 以及空字符串替换为 None，确保数据库写入兼容。"""
sanitized = [] sanitized = []
for row in records: for row in records:
sanitized.append({ sanitized.append({
k: (None if isinstance(v, float) and (math.isnan(v) or math.isinf(v)) else v) k: (None if (isinstance(v, float) and (math.isnan(v) or math.isinf(v)))
or (isinstance(v, str) and v.strip() == '')
else v)
for k, v in row.items() for k, v in row.items()
}) })
return sanitized return sanitized
...@@ -68,7 +164,7 @@ class CleaningRequest(BaseModel): ...@@ -68,7 +164,7 @@ class CleaningRequest(BaseModel):
class SavingRequest(BaseModel): class SavingRequest(BaseModel):
"""数据保存请求模型""" """数据保存请求模型"""
task_id: str task_id: str
table_name: str table_name: Optional[str] = None # 风控稽查任务已预设表名，可不传
# ==================== 业务逻辑 ==================== # ==================== 业务逻辑 ====================
...@@ -277,15 +373,30 @@ class DataCleaningService: ...@@ -277,15 +373,30 @@ class DataCleaningService:
if task_id not in self.cleaned_data_cache: if task_id not in self.cleaned_data_cache:
raise DatabaseException(f"任务 {task_id} 的清洗数据不存在或已过期（超过30分钟）") raise DatabaseException(f"任务 {task_id} 的清洗数据不存在或已过期（超过30分钟）")
cleaned_data = self.cleaned_data_cache[task_id]['data'] cached = self.cleaned_data_cache[task_id]
cleaned_data = cached['data']
# 优先使用缓存中预设的表名（风控稽查任务已写死 risk_audit_visit）
target_table = cached.get('table_name') or table_name
if not target_table:
raise DatabaseException("未指定目标表名，请在请求中传入 table_name")
# 将中文列名映射为数据库英文字段名，并强制转换各字段类型（仅对 risk_audit_visit 生效）
if target_table == "risk_audit_visit":
cleaned_data = [
_coerce_fengkong_row(
{FENGKONG_COLUMN_MAP[k]: v for k, v in row.items() if k in FENGKONG_COLUMN_MAP}
)
for row in cleaned_data
]
# 保存到数据库 # 保存到数据库
affected_rows = await self.db_handler.insert_data( affected_rows = await self.db_handler.insert_data(
table_name, target_table,
cleaned_data cleaned_data
) )
logger.info(f"[{task_id}] 成功保存 {affected_rows} 行数据到 {table_name}") logger.info(f"[{task_id}] 成功保存 {affected_rows} 行数据到 {target_table}")
# 清理缓存 # 清理缓存
del self.cleaned_data_cache[task_id] del self.cleaned_data_cache[task_id]
...@@ -416,35 +527,50 @@ class DataCleaningService: ...@@ -416,35 +527,50 @@ class DataCleaningService:
# ── 4. 合并为大宽表（内存，不写文件） ────────────────── # ── 4. 合并为大宽表（内存，不写文件） ──────────────────
self.progress_manager.update_progress( self.progress_manager.update_progress(
task_id, status="processing", progress=90, message="正在合并数据宽表..." task_id, status="processing", progress=85, message="正在合并数据宽表..."
) )
df_merged = pd.DataFrame(all_records, columns=STANDARD_COLUMNS) df_merged = pd.DataFrame(all_records, columns=STANDARD_COLUMNS)
merged_records = _sanitize_nan( logger.info(f"[{task_id}] 大宽表合并完成，共 {len(df_merged)} 条记录")
df_merged.where(pd.notna(df_merged), None).to_dict(orient="records")
# ── 5. 低价计算（从数据库读取价盘，回填低价字段） ────────
self.progress_manager.update_progress(
task_id, status="processing", progress=93, message="正在执行低价计算..."
)
import importlib.util, pathlib
_lp_spec = importlib.util.spec_from_file_location(
"low_price_calc",
pathlib.Path(__file__).parent / "core_py" / "1低价计算.py",
)
_lp_mod = importlib.util.module_from_spec(_lp_spec)
_lp_spec.loader.exec_module(_lp_mod)
df_final = await asyncio.to_thread(_lp_mod.transform, df_merged)
final_records = _sanitize_nan(
df_final.where(pd.notna(df_final), None).to_dict(orient="records")
) )
logger.info(f"[{task_id}] 大宽表合并完成，共 {len(merged_records)} 条记录") logger.info(f"[{task_id}] 低价计算完成，共 {len(final_records)} 条记录")
# ── 5. 写入内存缓存 ────────────────────────────────────── # ── 6. 写入内存缓存 ──────────────────────────────────────
self._evict_expired_cache() self._evict_expired_cache()
self.cleaned_data_cache[task_id] = { self.cleaned_data_cache[task_id] = {
"data": merged_records, "data": final_records,
"department": "风控稽查数据清洗", "department": "风控稽查数据清洗",
"created_at": datetime.now(), "created_at": datetime.now(),
"row_count": len(merged_records), "row_count": len(final_records),
"table_name": "risk_audit_visit",
} }
self.progress_manager.update_progress( self.progress_manager.update_progress(
task_id, status="completed", progress=100, task_id, status="completed", progress=100,
message=f"风控稽查数据清洗完成，共 {len(merged_records)} 条记录，等待前端确认", message=f"风控稽查数据清洗完成，共 {len(final_records)} 条记录，等待前端确认",
processed_count=len(merged_records) processed_count=len(final_records)
) )
return { return {
"task_id": task_id, "task_id": task_id,
"status": "completed", "status": "completed",
"message": "风控稽查数据清洗成功", "message": "风控稽查数据清洗成功",
"data_preview": merged_records[:5], "data_preview": final_records[:5],
"total_rows": len(merged_records), "total_rows": len(final_records),
} }
except Exception as e: except Exception as e:
...@@ -611,8 +737,8 @@ async def save_cleaned_data(request: SavingRequest): ...@@ -611,8 +737,8 @@ async def save_cleaned_data(request: SavingRequest):
Returns: { code, msg, data: { task_id, status, affected_rows } } Returns: { code, msg, data: { task_id, status, affected_rows } }
""" """
try: try:
if not request.task_id or not request.table_name: if not request.task_id:
return fail_resp(BizCode.BAD_REQUEST, "参数不完整：task_id 和 table_name 均为必填") return fail_resp(BizCode.BAD_REQUEST, "参数不完整：task_id 为必填")
result = await service.save_cleaned_data(request.task_id, request.table_name) result = await service.save_cleaned_data(request.task_id, request.table_name)
......
core_py/1低价计算.py deleted 100644 → 0
浏览文件 @ fa6c4c14
import pandas as pd
from datetime import datetime
from dateutil.relativedelta import relativedelta
# 文件路径
# TODO: 配置稽查月份（默认1号）
current_date = (datetime.now().replace(day=1) - relativedelta(months=1)).strftime("%Y-%m-01")
y_file = f"/王小卤/风控/代码-新/大日期{current_date}_2.xlsx"
p_file = f"/王小卤/风控/代码-新//线下价盘表2601版.xlsx"
# 保存回原文件（建议先保存为新文件以防覆盖）
output_file = f"/王小卤/风控/代码-新//低价大日期_2.xlsx"
# 读取Y表（稽查结果表）
df_y = pd.read_excel(y_file,sheet_name='合并后', dtype=str) # 先以字符串读入避免格式问题，后续转数字
# 读取P表（价盘表）
df_p = pd.read_excel(p_file, dtype=str)
# 清理列名（去除前后空格等）
df_y.columns = df_y.columns.str.strip()
df_p.columns = df_p.columns.str.strip()
# 将关键字段转换为统一格式（去除空格、统一大小写等，便于匹配）
def clean_str(s):
if pd.isna(s):
return ""
return str(s).strip().upper()
# 对Y表的关键列清洗
df_y['产品系列_clean'] = df_y.iloc[:, 14].apply(clean_str) # O列：产品系列
df_y['产品克重_clean'] = df_y.iloc[:, 16].apply(clean_str) # Q列：产品克重
df_y['渠道类型_clean'] = df_y.iloc[:, 13].apply(clean_str) # N列：渠道类型（稽查源提供）
# 对P表的关键列清洗
df_p['产品系统_clean'] = df_p.iloc[:, 0].apply(clean_str) # A列：产品系统
df_p['产品克重_p_clean'] = df_p.iloc[:, 2].apply(clean_str) # C列：产品克重
df_p['渠道_p_clean'] = df_p.iloc[:, 3].apply(clean_str) # D列：渠道
# 将价格列转为数值类型（注意处理非数字情况）
df_y['产品价格_num'] = pd.to_numeric(df_y.iloc[:, 17], errors='coerce') # R列：产品价格
df_p['低价_num'] = pd.to_numeric(df_p.iloc[:, 4], errors='coerce') # E列：低价
# 构建P表的唯一键（产品系统 + 产品克重 + 渠道）
df_p['match_key'] = df_p['产品系统_clean'] + '|' + df_p['产品克重_p_clean'] + '|' + df_p['渠道_p_clean']
# 构建Y表的匹配键（产品系列 + 产品克重 + 渠道类型）
df_y['match_key'] = df_y['产品系列_clean'] + '|' + df_y['产品克重_clean'] + '|' + df_y['渠道类型_clean']
# 将P表转为字典：key -> 低价
price_map = df_p.set_index('match_key')['低价_num'].to_dict()
# 初始化Y表的目标列（S: 是否低价, T: 破价价差）
df_y['是否低价'] = '正常' # 默认值
df_y['破价价差'] = None
# 遍历Y表每一行进行匹配和判断
for idx, row in df_y.iterrows():
key = row['match_key']
y_price = row['产品价格_num']
p_low_price = price_map.get(key, None)
if pd.notna(y_price) and pd.notna(p_low_price):
if y_price < p_low_price:
df_y.at[idx, '是否低价'] = '低价'
df_y.at[idx, '破价价差'] = round(p_low_price - y_price, 2)
df_y.at[idx, '低价整改状态'] = '未整改'
else:
df_y.at[idx, '是否低价'] = '正常'
df_y.at[idx, '破价价差'] = None
else:
# 无法匹配或价格缺失，保留默认或标记
df_y.at[idx, '是否低价'] = None
df_y.at[idx, '破价价差'] = None
# 只保留原始列（不保留清洗用的辅助列）
original_columns = df_y.columns.tolist()
output_columns = [col for col in original_columns if not col.endswith('_clean') and col not in ['产品价格_num', 'match_key']]
df_y[output_columns].to_excel(output_file, index=False)
print(f"处理完成！结果已保存至：{output_file}")
\ No newline at end of file
Markdown 格式
0% 或
您添加了 0 人 到此讨论。请谨慎行事。
请先完成此评论的编辑！
请 注册 或者 后发表评论