# 问题





预测模型执行代码过慢，想要快速预测。

每次都是将历史数据作为训练数据喂给模型，然后让模型输出预测数据。这个模型的训练过程太慢。

# 解决

（1）讲训练的模型存储起来，每次预测的时候直接用存储的模型进行预测，躲开训练环节；

（2）有新数据的时候，可以在不紧急的时候训练模型并存储起来；依照执行速度，可以设定更新频率，比如2个小时更新一次模型，1天更新一次模型。

# 实现

建立路径：

![在这里插入图片描述](/static/img/images/00331.png)

quikly_predict.py 内容
```python
import os
from fbprophet import Prophet
import pandas as pd
import matplotlib.pyplot as plt
import math
import pickle
class data_quikly_predict:
    pwd = os.path.dirname(os.path.abspath(__file__))
    data_dir = os.path.join(pwd, "data")
    def __init__(self):
        if not os.path.exists(self.data_dir):
            os.mkdir(self.data_dir)
        self.model_list = self.find_all_file(self.data_dir)
    def update_model_list(self):
        self.model_list = self.find_all_file(self.data_dir)
    def find_all_file(self, dirname):
        result = []
        for maindir, subdir, file_name_list in os.walk(dirname):
            for filename in file_name_list:
                apath = os.path.join(maindir, filename)
                result.append(apath)
        return result
    def create_new_model(self, df: pd.DataFrame, new_model_name: str):
        """
        :param df: 训练新模型所需要的数据
        :param new_model_name: 模型名称
        :return:
        """
        assert ("ds" in df.columns) and ("y" in df.columns)
        df['ds'] = df['ds'].astype('datetime64[ns]')
        model = Prophet(yearly_seasonality=True, weekly_seasonality=True)
        model.fit(df)  # 训练模型m
        model_file = os.path.join(self.data_dir, new_model_name)
        if model_file in self.model_list:
            print("模型名称已经被使用,但即将被置换\r\n")
        with open(model_file, 'wb') as f:
            pickle.dump(model, f)
        self.update_model_list()
    def predict(self, future_time: pd.DataFrame, model_name: str):
        """
        :param future_time:未来需要预测的时间点DataFrame，只有一行'ds'
        :param model_name:想要使用的模型名称
        :return:
        """
        model_file = os.path.join(self.data_dir, model_name)
        if model_file not in self.model_list:
            raise print("没有在data里找到这个模型文件，请确保用数据训练出了可使用的模型\r\n")
        with open(model_file, 'rb') as f:
            model = pickle.load(f)
            forecast = model.predict(future_time)  # 开始预测
            forecast = forecast[['ds', 'yhat']]
        return forecast
    def dummy_predict(self, periods, freq: str, model_name: str):
        """
        傻瓜化预测
        :param periods: 需要预测多少个数据结果
        :param freq: 从训练数据末尾调用频率
        :param model_name: 模型名称
        :return:
        """
        model_file = os.path.join(self.data_dir, model_name)
        if model_file not in self.model_list:
            raise print("没有在data里找到这个模型文件，请确保用数据训练出了可使用的模型\r\n")
        with open(model_file, 'rb') as f:
            model = pickle.load(f)
        future_time = model.make_future_dataframe(periods=periods, freq=freq, include_history=False)  # 预测的设置 还没预测
        forecast = model.predict(future_time)  # 开始预测
        forecast = forecast[['ds', 'yhat']]
        return forecast
    def quikly_plot_forecast(self, forecast: pd.DataFrame):
        plt.plot(list(forecast['ds']), list(forecast['yhat']), color='b')
        plt.show()
if __name__ == '__main__':
    dataqp = data_quikly_predict()
    timelist = list(pd.date_range(start='2021-01-01 00:00:00', end='2022-01-01 00:00:00', freq='H'))
    y = [math.sin(data.hour) for k, data in enumerate(timelist)]
    data_df = pd.DataFrame({'ds': timelist, 'y': y})
    dataqp.create_new_model(data_df, "camera1")
    future = pd.DataFrame({'ds': list(pd.date_range(start='2022-05-01 00:00:00',
                                                    end='2022-05-05 01:00:00',
                                                    freq='H'))})  # 预测的设置 还没预测
    forecast_res = dataqp.predict(future, model_name="camera1")
    dataqp.quikly_plot_forecast(forecast_res)
```

# 如何使用

下面代码是正常过程，先执行训练，然后预测数据。
```python
import math
import pandas as pd
from fbprophet_persistent_prodict.quikly_predict import data_quikly_predict
dataqp = data_quikly_predict()
timelist = list(pd.date_range(start='2021-01-01 00:00:00', end='2022-01-01 00:00:00', freq='H'))
y = [math.sin(data.hour) for k, data in enumerate(timelist)]
data_df = pd.DataFrame({'ds': timelist, 'y': y})
dataqp.create_new_model(data_df, "camera6")
future = pd.DataFrame({'ds': list(pd.date_range(start='2022-05-01 00:00:00',
                                            end='2022-05-05 01:00:00',
                                            freq='H'))})  # 预测的设置 还没预测
forecast_res = dataqp.predict(future, model_name="camera6")
dataqp.quikly_plot_forecast(forecast_res)
```

先预测，后用训练更新模型：

```csharp
import math
import pandas as pd
from fbprophet_persistent_prodict.quikly_predict import data_quikly_predict
dataqp = data_quikly_predict()
future = pd.DataFrame({'ds': list(pd.date_range(start='2022-05-01 00:00:00',
                                            end='2022-05-05 01:00:00',
                                            freq='H'))})  # 预测的设置 还没预测
forecast_res = dataqp.predict(future, model_name="camera6")
timelist = list(pd.date_range(start='2021-01-01 00:00:00', end='2022-01-01 00:00:00', freq='H'))
y = [math.sin(data.hour) for k, data in enumerate(timelist)]
data_df = pd.DataFrame({'ds': timelist, 'y': y})
dataqp.create_new_model(data_df, "camera6")
```

python 时序预测模型 fbprophet 持久化存储 大数据快速预测

利用python解释器对文件内置的__file__属性获取：




```python
os.path.dirname(os.path.abspath(__file__))
```

os.path.abspath(__file__)是当前py文件在操作系统的绝对路径文件名，os.path.dirname获取文件所在路径。

python 获取当前py文件的绝对路径

# 安装Prophet





python: 3.7.9

pystan: 2.19.0.0

pandas

fbprophet: 0.6.0

anaconda方式：

conda install pystan=2.19.0.0

conda install -c conda-forge fbprophet=0.6.0

# Prophet介绍

https://www.cnblogs.com/bonelee/p/9577432.html

https://facebook.github.io/prophet/docs/quick_start.html

可以预测数据，也可以给出趋势。时间序列预测上，充满专家经验：周期趋势、离群点、突变点、突变。

# 时间跨度

make_future_dataframe中的fre是Offset aliases形式的，用的pandas时间跨度

https://pandas.pydata.org/pandas-docs/stable/user_guide/timeseries.html#timeseries-offset-aliases

比如1hour1min可以表示时间跨度是61分钟

# 预测未来1

利用函数make_future_dataframe生成未来时间DataFrame对象future ：

```python
from fbprophet import Prophet
import pandas as pd
import matplotlib.pyplot as plt
import math
timelist = list(pd.date_range(start='2021-01-01 00:00:00', end='2022-01-01 00:00:00', freq='H'))
y = [math.sin(data.hour) for k, data in enumerate(timelist)]
data_df = pd.DataFrame({'ds': timelist, 'y': y})
data_df['ds'] = data_df['ds'].astype('datetime64[ns]')
m = Prophet()
m.fit(data_df)  # 训练模型m
future = m.make_future_dataframe(periods=50, freq='H',include_history=False)  # 预测的设置 还没预测
forecast = m.predict(future)  # 开始预测
# plt.plot(data_df['ds'][-50:], data_df['y'][-50:], color='b')
# plt.plot(forecast['ds'], forecast['yhat'], color='r')
# plt.show()
plt.plot(list(data_df['ds'][-50:])+list(forecast['ds']),
         list(data_df['y'][-50:])+list(forecast['yhat']), color='b')
plt.show()
```

![在这里插入图片描述](/static/img/images/00332.png)

# 预测未来2

未来任意时间：

```python
# 1 取数据
# https://www.kesci.com/mw/dataset/5d64a35a8499bc002c07bb1b
from fbprophet import Prophet
import pandas as pd
import matplotlib.pyplot as plt
import math
timelist = list(pd.date_range(start='2021-01-01 00:00:00', end='2022-01-01 00:00:00', freq='H'))
y = [math.sin(data.hour) for k, data in enumerate(timelist)]
data_df = pd.DataFrame({'ds': timelist, 'y': y})
data_df['ds'] = data_df['ds'].astype('datetime64[ns]')
m = Prophet()
m.fit(data_df)  # 训练模型m
future = pd.DataFrame({'ds': list(pd.date_range(start='2022-05-01 00:00:00', end='2022-05-05 00:00:00', freq='H'))})  # 预测的设置 还没预测
forecast = m.predict(future)  # 开始预测
# plt.plot(data_df['ds'][-50:], data_df['y'][-50:], color='b')
plt.plot(forecast['ds'], forecast['yhat'], color='r')
plt.show()
# plt.plot(list(data_df['ds'][-50:])+list(forecast['ds']),
#          list(data_df['y'][-50:])+list(forecast['yhat']), color='b')
# plt.show()
```


![在这里插入图片描述](/static/img/images/00333.png)

# 预测模型持久化存储

时序预测模型 fbprophet

直接dict即可：





```python
>>> aa = [("kevin", 100,), ("dylan", 100,), ("daisy", 100,)]
>>> dict(aa)
{'kevin': 100, 'dylan': 100, 'daisy': 100}
```

引申

只要是字典结构的数据都能这么玩，比如：

```python
aa = {("kevin", 100,), ("dylan", 100,), ("daisy", 100,)}
aa = [["kevin", 100,], ["dylan", 100,], ["daisy", 100,]]
```

python 列表元组 转 字典

# 参考





python的re模块说明：

[https://docs.python.org/zh-cn/3/library/re.html](https://docs.python.org/zh-cn/3/library/re.html)

一些常见的正则表达式写法（邮箱、手机号、密码、身份证）：

[https://zhuanlan.zhihu.com/p/338826624](https://zhuanlan.zhihu.com/p/338826624)

# 使用

## re.findall

非常有用！

![在这里插入图片描述](/static/img/images/00334.png)

```powershell
in_str = """
Name: 10.10.90.86:9866 (hadoop001)
DFS Used: 149719834624 dgf
Name: 10.10.90.87:9866 (hadoop002)
DFS Used: 1497198231234624 dgfg
Name: 10.10.90.88:9866 (hadoop003)
DFS Used: 14971981323134624 gfdg
"""
import re
pattern1 = re.compile(r"Name: [0-9]+.[0-9]+.[0-9]+.[0-9]+")
dalllist1 = re.findall(pattern1, in_str)
dalllist1 = list(map(lambda x: x[6:], dalllist1))
pattern2 = re.compile(r"DFS Used: [0-9]+")
dalllist2 = re.findall(pattern2, in_str)
dalllist2 = list(map(lambda x: x[14:], dalllist2))
for i in zip(dalllist1, dalllist2):
    print(i)
```

## re.search

感觉基本没啥用。

![在这里插入图片描述](/static/img/images/00335.png)

```powershell
in_str = """
Name: 10.10.90.86:9866 (hadoop001)
DFS Used: 149719834624 dgf
Name: 10.10.90.87:9866 (hadoop002)
DFS Used: 1497198231234624 dgfg
Name: 10.10.90.88:9866 (hadoop003)
DFS Used: 14971981323134624 gfdg
"""
import re
pattern1 = re.compile(r"Name: [0-9]+.[0-9]+.[0-9]+.[0-9]+")
dalllist1 = re.search(pattern1, in_str)
print(dalllist1.group())
```

## re.match

用这个来匹配密码很好，用户设置密码的时候必须有数字和大小写之类的。

![在这里插入图片描述](/static/img/images/00336.png)

```powershell
import re
while True:
    pwd = input("input your password\r\n")
    if re.match(re.compile("^[a-z]{8,16}$"), pwd) == None:
        print("no pass")
    else:
        print("password is ok")
```

密码强度：[https://www.huaweicloud.com/articles/e43ea3c29dfbebc830af0e0b69b71ce3.html](https://www.huaweicloud.com/articles/e43ea3c29dfbebc830af0e0b69b71ce3.html)

## re.split

```powershell
import re
text = """Ross McFluff: 834.345.1254 155 Elm Street
...
... Ronald Heathmore: 892.345.3428 436 Finley Avenue
... Frank Burger: 925.541.7625 662 South Dogwood Way
...
...
... Heather Albrecht: 548.326.4584 919 Park Place"""
entries = re.split("\n+", text)
print([re.split(":? ", entry, 3) for entry in entries])
```


# 一些别的

> []

匹配方括号里的所有字符的集合。集合set的意味在里面。

比如`[34]`就是匹配3或者匹配4。

比如`[A-Za-z]` 就是匹配所有的大写字母或者小写字母。

比如`[0-9]` 就是匹配所有的数字。

  

> ^

如果放到 [  ]  里面，就表示这个集合里能匹配的东西都不是想匹配的。

比如`[^heo]`就表示匹配除了这三个字母集合外的所有字符集合。

如果放到 [  ]  外面，就表示匹配一个行开始。

比如`^[heo]`就表示匹配某一个新行，新行以三个字母的某一个字母开头。

要匹配 ^ 字符本身，请使用 `\^`。

> $

匹配行结尾。这符号很特殊。是一个零长度字符的匹配。

> .

匹配除换行符 \n 之外的任何单字符。要匹配 . ，请使用 `\.` 。

python 正则表达式 教程 使用 查找字符串 搜索是否含有 匹配密码强度 分割出合适数据

身在Docker容器里身不由己，wget能很好解决：





下载：
```powershell
wget https://repo.anaconda.com/miniconda/Miniconda3-py38_4.9.2-Linux-x86_64.sh
```

安装：

-b是静默安装，无需交互。
```powershell
sh Miniconda3-py38_4.9.2-Linux-x86_64.sh -b
```

所以记住这2个仓库：

[https://repo.anaconda.com/miniconda/](https://repo.anaconda.com/miniconda/)

[https://repo.anaconda.com/archive/](https://repo.anaconda.com/archive/)

Linux wget miniconda 快速安装miniconda环境

参考：[https://zhuanlan.zhihu.com/p/108418193](https://zhuanlan.zhihu.com/p/108418193)





![在这里插入图片描述](/static/img/images/00360.png)

什么时候需要定义类 而不是函数？

python安装opencv：





> pip install opencv-python

提取视频帧 视频转图片：
```python
import os
mp4file = 'input.mp4'
savefile = "./output"
if not os.path.exists(savefile):
    os.mkdir(savefile)
import cv2
cap = cv2.VideoCapture(mp4file)
fourcc = cv2.VideoWriter_fourcc(*'XVID')
fps = cap.get(cv2.CAP_PROP_FPS)
size = (int(cap.get(cv2.CAP_PROP_FRAME_WIDTH)), int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT)))
print(fps)
print(size)
i = 0
while (cap.isOpened()):
    i = i + 1
    ret, frame = cap.read()
    if ret == True:
        cv2.imwrite(savefile + '/' + str(i) + '.jpg', frame)
        if cv2.waitKey(1) & 0xFF == ord('q'):
            break
    else:
        break
print(i)
cap.release()
cv2.destroyAllWindows()
```

图片转视频：

```python
import os
mp4file = 'out.mp4'
savefile = "./output"
if not os.path.exists(savefile):
    os.mkdir(savefile)
import cv2
fourcc = cv2.VideoWriter_fourcc(*'XVID')
fps = 30
size =(720, 1280)
videoWriter = cv2.VideoWriter(mp4file, fourcc, fps, size)  # 最后一个是保存图片的尺寸
for i in range(1, 1402):
    frame = cv2.imread(savefile + '/' + str(i) + '.jpg')
    videoWriter.write(frame)
videoWriter.release()
```


合成，一定得注意尺寸的大小。

```python
# coding=utf-8
import os
import cv2
from PIL import Image
def makevideo(path, fps):
    """ 将图片合成视频. path: 视频路径，fps: 帧率 """
    fourcc = cv2.VideoWriter_fourcc(*"mp4v")
    path1 = r'D:\yifei111\PycharmProjects\python_test\jigan_res4'
    im = Image.open(r'D:\yifei111\PycharmProjects\python_test\jigan1\251.jpg')
    print(im.size)
    vw = cv2.VideoWriter(path, fourcc, fps, im.size)
    for i in os.listdir(path1):
        frame = cv2.imread(path1 + '\\' + i)
        frame = frame[200:649, :]
        frame = cv2.resize(frame, im.size)
        vw.write(frame)
if __name__ == '__main__':
    video_path = r'D:\yifei111\PycharmProjects\python_test\DST_test_new4.mp4'
    makevideo(video_path, 30)  # 图片转视频
```

python opencv 提取视频帧 视频转图片 图片转视频

借助pd.date_range快速实现





## 生成 Timestamp对象 列表

生成一系列Timestamp对象，Timestamp对象是对python datetime再封装得到的，一般情况直接用起来就好了。
```python
import pandas as pd
a = list(pd.date_range(start='2021-06-21 00:00:00', end='2021-06-22 00:00:00', freq='H'))
print(a)
```

得到：

[Timestamp('2021-06-21 00:00:00', freq='H'), Timestamp('2021-06-21 01:00:00', freq='H'), Timestamp('2021-06-21 02:00:00', freq='H'), Timestamp('2021-06-21 03:00:00', freq='H'), Timestamp('2021-06-21 04:00:00', freq='H'), Timestamp('2021-06-21 05:00:00', freq='H'), Timestamp('2021-06-21 06:00:00', freq='H'), Timestamp('2021-06-21 07:00:00', freq='H'), Timestamp('2021-06-21 08:00:00', freq='H'), Timestamp('2021-06-21 09:00:00', freq='H'), Timestamp('2021-06-21 10:00:00', freq='H'), Timestamp('2021-06-21 11:00:00', freq='H'), Timestamp('2021-06-21 12:00:00', freq='H'), Timestamp('2021-06-21 13:00:00', freq='H'), Timestamp('2021-06-21 14:00:00', freq='H'), Timestamp('2021-06-21 15:00:00', freq='H'), Timestamp('2021-06-21 16:00:00', freq='H'), Timestamp('2021-06-21 17:00:00', freq='H'), Timestamp('2021-06-21 18:00:00', freq='H'), Timestamp('2021-06-21 19:00:00', freq='H'), Timestamp('2021-06-21 20:00:00', freq='H'), Timestamp('2021-06-21 21:00:00', freq='H'), Timestamp('2021-06-21 22:00:00', freq='H'), Timestamp('2021-06-21 23:00:00', freq='H'), Timestamp('2021-06-22 00:00:00', freq='H')]

## 生成 str对象 列表

```python
import pandas as pd
a = list(pd.date_range(start='2021-06-21 00:00:00', end='2021-06-22 00:00:00', freq='H'))
a=list(map(str,a))
print(a)
```

得到：

['2021-06-21 00:00:00', '2021-06-21 01:00:00', '2021-06-21 02:00:00', '2021-06-21 03:00:00', '2021-06-21 04:00:00', '2021-06-21 05:00:00', '2021-06-21 06:00:00', '2021-06-21 07:00:00', '2021-06-21 08:00:00', '2021-06-21 09:00:00', '2021-06-21 10:00:00', '2021-06-21 11:00:00', '2021-06-21 12:00:00', '2021-06-21 13:00:00', '2021-06-21 14:00:00', '2021-06-21 15:00:00', '2021-06-21 16:00:00', '2021-06-21 17:00:00', '2021-06-21 18:00:00', '2021-06-21 19:00:00', '2021-06-21 20:00:00', '2021-06-21 21:00:00', '2021-06-21 22:00:00', '2021-06-21 23:00:00', '2021-06-22 00:00:00']

## 生成 python datetime对象 列表

```python
import pandas as pd
a = list(pd.date_range(start='2021-06-21 00:00:00', end='2021-06-22 00:00:00', freq='H'))
a=list(map(lambda x:x.to_pydatetime(),a))
print(a)
```

得到：

```python
[datetime.datetime(2021, 6, 21, 0, 0), datetime.datetime(2021, 6, 21, 1, 0), datetime.datetime(2021, 6, 21, 2, 0), datetime.datetime(2021, 6, 21, 3, 0), datetime.datetime(2021, 6, 21, 4, 0), datetime.datetime(2021, 6, 21, 5, 0), datetime.datetime(2021, 6, 21, 6, 0), datetime.datetime(2021, 6, 21, 7, 0), datetime.datetime(2021, 6, 21, 8, 0), datetime.datetime(2021, 6, 21, 9, 0), datetime.datetime(2021, 6, 21, 10, 0), datetime.datetime(2021, 6, 21, 11, 0), datetime.datetime(2021, 6, 21, 12, 0), datetime.datetime(2021, 6, 21, 13, 0), datetime.datetime(2021, 6, 21, 14, 0), datetime.datetime(2021, 6, 21, 15, 0), datetime.datetime(2021, 6, 21, 16, 0), datetime.datetime(2021, 6, 21, 17, 0), datetime.datetime(2021, 6, 21, 18, 0), datetime.datetime(2021, 6, 21, 19, 0), datetime.datetime(2021, 6, 21, 20, 0), datetime.datetime(2021, 6, 21, 21, 0), datetime.datetime(2021, 6, 21, 22, 0), datetime.datetime(2021, 6, 21, 23, 0), datetime.datetime(2021, 6, 22, 0, 0)]
```

# 转换

 pd.Timestamp 转 datetime.datetime  ：
```python
pd.Timestamp("2021-06-06").to_pydatetime()
```

datetime.datetime  转 pd.Timestamp：

```python
pd.to_datetime(datetime.datetime.now())
```

python 生成一段日期列表 pd.date_range Timestamp 转 datetime datetime 转 Timestamp

（1）本质核心是多进程里执行os.system("wget filename")，开17个进程的进程池意味着同时在下载17个文件。





（2）将指令全部放列表needgfsfile里面。

```python
import datetime
import multiprocessing
import os
def downloadgfs(filename):
    os.system(filename)
start_hour = 12
yesterday = (datetime.datetime.now() - datetime.timedelta(days=1)).replace(hour=start_hour, microsecond=0)
ymdstr = f"{str(yesterday.year).zfill(4)}{str(yesterday.month).zfill(2)}{str(yesterday.day).zfill(2)}{str(start_hour).zfill(2)}"
if not os.path.exists(ymdstr):
    os.mkdir(ymdstr)
needgfsfile = [
    f"wget ftp://ftpprd.ncep.noaa.gov/pub/data/nccf/com/gfs/prod/gfs.{ymdstr[:-2]}/{ymdstr[-2:]}/atmos/gfs.t{ymdstr[-2:]}z.pgrb2.0p50.f{str(i * 24).zfill(3)} -P {ymdstr}/"
    for i in range(17)]
p = multiprocessing.Pool(processes=17)  # 进to程池
_ = [p.apply_async(func=downloadgfs, args=(i,)) for i in needgfsfile]
p.close()
p.join()
```