def Train_func(qfile,train_data,max_circle=10000):
mySetting = Settings()
maxDays = max_circle
# 创建一个空列表ProductList，用于存储产品对象
ProductList = []

# 在给定的最大天数maxDays范围内反复遍历训练数据集中的每一天
step = len(train_data)
preAction = 0
preNum = 0
for i in range(maxDays):
# 机器学习，补货
j = i%step # 将变量i除以step的余数赋值给变量j,将i限定在0到step-1之间，循环遍历train_data
buy_num = int(train_data[j])
# int将字符串型数据转换为整数型数据
# 从训练数据集中获取第j天的购买数buy_num

# 清理过期货物
overdue_n = getOverDueNum(ProductList)
deleteOverDueProduct(ProductList)

# 当前状态
current_num = getInStoreHouseNum(ProductList)
if buy_num > current_num:
short_n = buy_num - current_num
else:
short_n = 0

# 当天卖出的货物出库，即将前buy_num - short_n个产品从列表中删除
ProductList = CheckOutProduct(ProductList, buy_num - short_n)
# 重新获取当前状态，准备补货
current_num = getInStoreHouseNum(ProductList)

# 根据贝尔曼方程，更新 Q table 中当前状态-动作对应的 Q value
alpha = mySetting.getAlpha(i)
gamma = mySetting.getGamma()
rewards = mySetting.getRewards(overdue_n,short_n,buy_num - short_n,preNum)
pre_q_v = mySetting.Q_table[preNum, preAction]
mySetting.Q_table[preNum, preAction] += alpha * (rewards + gamma * max(mySetting.Q_table[current_num]) - pre_q_v)

# 随机或贪婪地选择当前动作
if (np.random.uniform() < mySetting.getEpsilon()) or ((mySetting.Q_table[current_num, :] == mySetting.init_min_val).all()):
tmp_a = get_valid_actions(mySetting, current_num)
s = random.randint(0,len(tmp_a)-1)
current_action = mySetting.actions.index(tmp_a[s]) # 探索
else:
valid_actions = [a for a in mySetting.actions if (a * mySetting.unit + current_num) <= mySetting.max_storages]
current_action = mySetting.Q_table[current_num, :len(valid_actions)].argmax() # 利用

preNum = current_num
preAction = current_action

# 补充货物
add_num = mySetting.actions[current_action] * mySetting.unit
AddProductOrder(ProductList,mySetting.liftime,mySetting.onwaydays,add_num)

# 刷新货物生存时间
updateProductLife(ProductList)

fh = open(qfile,“w”)
aa = mySetting.Q_table.shape
# 遍历Q_table数组中的每个元素，将其转化为字符串形式
for i in range(aa[0]):
tmp1 = mySetting.Q_table[i].tolist()
# 按照逗号进行连接，并在其末尾添加一个换行符
tmp2 = [str(a) for a in tmp1]
cntStr = “,”.join(tmp2)+“\n”
fh.write(cntStr)
fh.close()

def Exe_func1(qfile,outfile,logfile,test_data,init_num):
mySetting = Settings()
ProductList = []
mySetting.Q_table = Read_Q_Data(qfile)

AddProductOrder(ProductList, mySetting.liftime,mySetting.onwaydays, init_num)
updateProductLife(ProductList)
updateProductLife(ProductList)
updateProductLife(ProductList)

lgf = open(logfile,“w”)
# 向文件中写入第一行标题，包括需求日期、需求量等信息
lgf.write(“日期,需求,存量,过期量,缺货,订货动作,库存变动\n”)

fh = open(outfile,“w”)
step = len(test_data)
for i in range(step):
buy_num = int(test_data[i]) # 从测试集中获取第i天的购买数buy_num

# 清理过期货物
overdue_n = getOverDueNum(ProductList)
deleteOverDueProduct(ProductList)

# 当前状态
current_num = getInStoreHouseNum(ProductList)
storage_num = current_num
if buy_num > current_num:
short_n = buy_num - current_num
else:
short_n = 0

# 当天卖出的货物出库，即将前buy_num - short_n个产品从列表中删除
ProductList = CheckOutProduct(ProductList, buy_num - short_n)
# 重新获取当前状态，准备补货
current_num = getInStoreHouseNum(ProductList)

valid_actions = [a for a in mySetting.actions if (a * mySetting.unit + current_num) <= mySetting.max_storages]
current_action = mySetting.Q_table[current_num, :len(valid_actions)].argmax() # 利用
# 补充货物
add_num = mySetting.actions[current_action] * mySetting.unit
AddProductOrder(ProductList,mySetting.liftime,mySetting.onwaydays,add_num)

# 刷新货物生存时间
updateProductLife(ProductList)

store_cost = mySetting.get_store_cost(overdue_n,short_n,current_num+add_num)
cntStr = “{0},{1:.2f}\n”.format(i, store_cost)
fh.write(cntStr)

loginfo =[str(i),str(buy_num),str(storage_num),str(overdue_n),str(short_n),str(add_num),str(current_num+add_num-storage_num)]
cntStr = “,”.join(loginfo) + “\n”
lgf.write(cntStr)

fh.close()
lgf.close()

def Exe_func2(qfile,outfile,logfile,test_data,init_num):
mySetting = Settings()
ProductList = []
mySetting.Q_table = Read_Q_Data(qfile)

AddProductOrder(ProductList, mySetting.liftime,mySetting.onwaydays, init_num)
updateProductLife(ProductList)
updateProductLife(ProductList)
updateProductLife(ProductList)

lgf = open(logfile,“w”)
# 向文件中写入第一行标题，包括需求日期、需求量等信息
lgf.write(“日期,需求,存量,过期量,缺货,订货动作,库存变动\n”)

fh = open(outfile,“w”)
step = len(test_data)
for i in range(step):
buy_num = int(test_data[i]) # 从测试集中获取第i天的购买数buy_num

# 清理过期货物
overdue_n = getOverDueNum(ProductList)
deleteOverDueProduct(ProductList)

# 当前状态
current_num = getInStoreHouseNum(ProductList)
storage_num = current_num
if buy_num > current_num:
short_n = buy_num - current_num
else:
short_n = 0

# 当天卖出的货物出库，即将前buy_num - short_n个产品从列表中删除
ProductList = CheckOutProduct(ProductList, buy_num - short_n)
# 重新获取当前状态，准备补货
current_num = getInStoreHouseNum(ProductList)

add_num = 0
if current_num<mySetting.order_point:
add_num = mySetting.order_volume

AddProductOrder(ProductList, mySetting.liftime, mySetting.onwaydays, add_num)
updateProductLife(ProductList)

store_cost = mySetting.get_store_cost(overdue_n, short_n,current_num+add_num)
cntStr = “{0},{1:.2f}\n”.format(i, store_cost)
fh.write(cntStr)

loginfo = [str(i),str(buy_num),str(storage_num),str(overdue_n),str(short_n),str(add_num),str(current_num+add_num-storage_num)]
cntStr = “,”.join(loginfo) + “\n”
lgf.write(cntStr)

fh.close()
lgf.close()

# 读取文件所有的行
def ReadTxtName(filename, dot=‘,’, encoding=“gbk”):
lines = []
if os.path.exists(filename): # 判断filename文件是否存在
pass
else:
return []

with open(filename, ‘r’, encoding=encoding) as file_to_read:
tmpall = file_to_read.readlines()
for line in tmpall:
tmpstr = line.strip(‘\n’).strip() # 去掉每行字符串开头和结尾的空白字符（包括空格、制表符、换行符等）
if tmpstr.startswith(‘#’): # 判断tmpstr字符串是否以#开头
continue # 如果是注释行，则继续跳过后面的代码，进行下一轮循环

if len(tmpstr) >0:
line = tmpstr.split(dot) # 将tmpstr字符串按照指定的分隔符dot进行分割
lines.append(line)
return lines

# list为二维列表，row为行，col为列
def GetColumnFromList(list, startrow=0, endrow=None, col=0):
data = []
if endrow is None or endrow > len(list):
endrow = len(list)

for i in range(startrow, endrow): # 遍历列表的行数
data.append(list[i][col]) # 将每行的第“col”列的数据添加到“data”列表

return data

# 判断传入的参数是否为数值类型，如果是就返回 True，否则不返回结果
def is_number(num):
if isinstance(num, str):
pass
else:
return True

# 正则表达式，用于匹配数值类型的字符串形式
pattern = re.compile(r’[1]?[-0-9]\d*.\d*|[-+]?.?[0-9]\d*$‘)
result = pattern.match(num)

if result:
return True
else:
return False

# 将字符串列表转化为浮点数列表
def StrListToFloat(list):
data = []
for x in list:
if x != ‘’: # 当前元素不为空字符串
data.append(float(x)) # 转化为浮点数类型
else:
data.append(0)
return data

# 将二维字符串列表转化为浮点数列表
def StrList2ToFloat(list):
rslt = []
for row in list:
data = []
for x in row:
if x != ‘’:
if is_number(x):
data.append(float(x))
else:
data.append(x)
# 如果当前元素为空字符串，则将其填补为默认值 0，并添加到 “data” 列表中
else:
data.append(0)
rslt.append(data)
return rslt

# 绘制出折线图
def DrawLine(logfile1,logfile2):
fig = plt.figure(figsize=(10, 5), dpi=80)
mng = plt.get_current_fig_manager()
left, width = 0.05, 0.90
# 主轴（ax1）的位置和大小，背景颜色
ax1 = fig.add_axes([left, 0.05, width, 0.90], gid=‘mygroup’)
ax1.set_facecolor(“White”)

# 获取logfile1中的第1列，并取绝对值
lines1 = ReadTxtName(logfile1, “,”)
day_col1 = GetColumnFromList(lines1, col=0)
day_col1 = day_col1[:]
day_col1 = StrListToFloat(day_col1)
day_col1 = [abs(aa) for aa in day_col1]

# 获取logfile1中的第2列，并取绝对值
loss_col1 = GetColumnFromList(lines1, col=1)
loss_col1 = loss_col1[:]
loss_col1 = StrListToFloat(loss_col1)
loss_col1 = [abs(aa) for aa in loss_col1]

ax1.plot(day_col1, loss_col1, color = ‘green’, linestyle=’-‘)

# 获取logfile2中的第1列，并取绝对值
lines2 = ReadTxtName(logfile2, “,”)
day_col2 = GetColumnFromList(lines2, col=0)
day_col2 = day_col2[:]
day_col2 = StrListToFloat(day_col2)
day_col2 = [abs(aa) for aa in day_col2]

# 获取logfile2中的第2列，并取绝对值
loss_col2 = GetColumnFromList(lines2, col=1)
loss_col2 = loss_col2[:]
loss_col2 = StrListToFloat(loss_col2)
loss_col2 = [abs(aa) for aa in loss_col2]

ax1.plot(day_col2, loss_col2, color = ‘r’, linestyle=’-')

ylabels = [0,10000,20000,30000,40000,50000,60000,70000,80000] # 设置y轴刻度标签
ax1.set_yticklabels(labels=ylabels,family= ‘SimHei’,fontsize= 12)
ax1.tick_params(which=‘major’,axis=‘both’,length=8,labelsize=12)
ax1.legend(labels=(‘Q-Learning 成本’, “固定订货成本”), loc=‘upper right’,fontsize=14)
plt.show()
matplotlib.rcParams[‘lines.linewidth’] = linewidth_bak # 恢复matplotlib默认设置

# 生成训练数据并保存到文件
def Create_Train_Data(datafile):
mysetting = Settings()
mu = mysetting.expect
sigma = mysetting.variance
num = mysetting.train_data_n
np.random.seed(1234)
Train_Data = np.random.normal(mu, sigma, num)
# 生成符合正态分布的随机数（μ为期望，σ为方差，数量为num）

f1 = open(datafile,“w”)
for i in range(len(Train_Data)):
cnt = int(Train_Data[i]+0.5)

if i<len(Train_Data)-1:
f1.write(str(cnt)+“,”)
else:
f1.write(str(cnt))
f1.close()

# 读取训练数据并返回其中的一行数据
def Read_Train_Data(datafile):
data = ReadTxtName(datafile)
return data[0]

# 读取Q值文件并返回二维列表
def Read_Q_Data(datafile):
data = ReadTxtName(datafile)
data1 = StrList2ToFloat(data)
rslt = np.array(data1)
return rslt

if name == “main”:
datafile = “Train_Data35.csv”
Create_Train_Data(datafile)

train_data = Read_Train_Data(datafile)

qfile = “qfile35.csv”
Train_func(qfile,train_data,200000)

rsltfile1 = “rslt35.csv”
logfile1 = “logfile35.csv”

Exe_func1(qfile,rsltfile1,logfile1,train_data,100)

rsltfile2 = “rslt35_1.csv”
logfile2 = “logfile35_1.csv”
Exe_func2(qfile,rsltfile2,logfile2,train_data,100)
DrawLine(rsltfile1,rsltfile2)
以上代码是内容还是格式出问题了？麻烦纠正