【FATE联邦学习】FATE框架里jupyter对应的save to fate函数在普通python代码里面对应的函数是什么呀？

Question

save to fate not working, please check if your ipython is installed, and if ipython.get_ipython() is working
  File "/root/Downloads/project_demo/mnist_example.py", line 44, in <module>
    save_to_fate()
ValueError: save to fate not working, please check if your ipython is installed, and if ipython.get_ipython() is working

官网文档 https://fate.readthedocs.io/en/latest/tutorial/

Answer 1

save to fate not working, please check if your ipython is installed, and if ipython.get_ipython() is working
  File "/root/Downloads/project_demo/mnist_example.py", line 44, in <module>
    save_to_fate()
ValueError: save to fate not working, please check if your ipython is installed, and if ipython.get_ipython() is working

你可能并不是用jupyter在搞官方那个demo。

意思是你可以用朱庇特的这个函数，也可以不用。但是他官方demo就是用的jupyter，具有误导性。我来重新写一个这个官方的demo，不使用这个函数的。
他说的copy code file to the directory中的directory，应该是federatedml/nn/dataset这个路径
比如这个image.py里面定义了一个数据集
可能他的意思是，让我们把我们上面那些定义mnist dataset的的定义新建一个文件mnist_dataset.py在这个路径里面。

因为如果不手动的话，本地的训练是可以的，但是下一部分的在FATE服务上会一直报错没有这个federeatedml.nn.dataset.mnist_dataset。
那就手动呗，你挑的嘛。注意这个的名字要和你代码里的nn那里相对应

那个文件里是以下代码

import torch
from federatedml.nn.dataset.base import Dataset
from torchvision.datasets import ImageFolder
from torchvision import transforms
import numpy as np
# 这里的包缺什么补什么哈

class MNISTDataset(Dataset):
    
    def __init__(self, flatten_feature=False): # flatten feature or not 
        super(MNISTDataset, self).__init__()
        self.image_folder = None
        self.ids = None
        self.flatten_feature = flatten_feature
        
    def load(self, path):  # read data from path, and set sample ids
        # read using ImageFolder
        self.image_folder = ImageFolder(root=path, transform=transforms.Compose([transforms.ToTensor()]))
        # filename as the image id
        ids = []
        for image_name in self.image_folder.imgs:
            ids.append(image_name[0].split('/')[-1].replace('.jpg', ''))
        self.ids = ids
        return self

    def get_sample_ids(self):  # implement the get sample id interface, simply return ids
        return self.ids
    
    def __len__(self,):  # return the length of the dataset
        return len(self.image_folder)
    
    def __getitem__(self, idx): # get item
        ret = self.image_folder[idx]
        if self.flatten_feature:
            img = ret[0][0].flatten() # return flatten tensor 784-dim
            return img, ret[1] # return tensor and label
        else:
            return ret


if __name__ == '__main__':
    pass

这样就完成了他所谓的“手动添加”了。jupyter的save to fate函数就是干了这些事儿。

修改后的其实没怎么边代码都是拷贝的，只不使用那个函数。其实就是把你新定义的那个数据集加载进来就是了，直接把save to fate那些代码删掉就行了

import os
# os.environ["PATH"] += os.pathsep + '/root/Downloads/standalone_fate_install_1.10.0_release/python'


import numpy as np
from torchvision.datasets import ImageFolder
from torchvision import transforms
from federatedml.nn.dataset.base import Dataset

class MNISTDataset(Dataset):
    
    def __init__(self, flatten_feature=False): # flatten feature or not 
        super(MNISTDataset, self).__init__()
        self.image_folder = None
        self.ids = None
        self.flatten_feature = flatten_feature
        
    def load(self, path):  # read data from path, and set sample ids
        # read using ImageFolder
        self.image_folder = ImageFolder(root=path, transform=transforms.Compose([transforms.ToTensor()]))
        # filename as the image id
        ids = []
        for image_name in self.image_folder.imgs:
            ids.append(image_name[0].split('/')[-1].replace('.jpg', ''))
        self.ids = ids
        return self

    def get_sample_ids(self):  # implement the get sample id interface, simply return ids
        return self.ids
    
    def __len__(self,):  # return the length of the dataset
        return len(self.image_folder)
    
    def __getitem__(self, idx): # get item
        ret = self.image_folder[idx]
        if self.flatten_feature:
            img = ret[0][0].flatten() # return flatten tensor 784-dim
            return img, ret[1] # return tensor and label
        else:
            return ret

# 这些直接删掉不要
# from pipeline.component.nn import save_to_fate
# save_to_fate()

ds = MNISTDataset(flatten_feature=True)
ds.load('mnist/')
print(len(ds))
print(ds[0])
print(ds.get_sample_ids()[0])


# test local process
from federatedml.nn.homo.trainer.fedavg_trainer import FedAVGTrainer
trainer = FedAVGTrainer(epochs=3, batch_size=256, shuffle=True, data_loader_worker=8, pin_memory=False) # set parameter

trainer.local_mode() 

import torch as t
from pipeline import fate_torch_hook
fate_torch_hook(t)
# our simple classification model:
model = t.nn.Sequential(
    t.nn.Linear(784, 32),
    t.nn.ReLU(),
    t.nn.Linear(32, 10),
    t.nn.Softmax(dim=1)
)

trainer.set_model(model) # set model

optimizer = t.optim.Adam(model.parameters(), lr=0.01)  # optimizer
loss = t.nn.CrossEntropyLoss()  # loss function
trainer.train(train_set=ds, optimizer=optimizer, loss=loss)  # use dataset we just developed


# exit() 后面的都一样，不再重复写了